Ensures metadata only is cached for pods and services

Signed-off-by: irbekrm <irbekrm@gmail.com>

Author: irbekrm

cmd/controller/app/controller.go

@@ -211,6 +211,7 @@ func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) error {
 	log.V(logf.DebugLevel).Info("starting shared informer factories")
 	ctx.SharedInformerFactory.Start(rootCtx.Done())
 	ctx.KubeSharedInformerFactory.Start(rootCtx.Done())
+	ctx.MetadataInformerFactory.Start(rootCtx.Done())
 
 	if utilfeature.DefaultFeatureGate.Enabled(feature.ExperimentalGatewayAPISupport) {
 		ctx.GWShared.Start(rootCtx.Done())

internal/informers/core.go

@@ -19,7 +19,6 @@ package informers
 import (
 	corev1 "k8s.io/api/core/v1"
 	certificatesv1 "k8s.io/client-go/informers/certificates/v1"
-	corev1informers "k8s.io/client-go/informers/core/v1"
 	networkingv1informers "k8s.io/client-go/informers/networking/v1"
 	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
@@ -45,8 +44,6 @@ const pleaseOpenIssue = "Please report this by opening an issue with this error
 type KubeInformerFactory interface {
 	Start(<-chan struct{})
 	WaitForCacheSync(<-chan struct{}) map[string]bool
-	Pods() corev1informers.PodInformer
-	Services() corev1informers.ServiceInformer
 	Ingresses() networkingv1informers.IngressInformer
 	Secrets() SecretInformer
 	CertificateSigningRequests() certificatesv1.CertificateSigningRequestInformer

internal/informers/core_basic.go

@@ -64,14 +64,6 @@ func (bf *baseFactory) WaitForCacheSync(stopCh <-chan struct{}) map[string]bool
 	return ret
 }
 
-func (bf *baseFactory) Pods() corev1informers.PodInformer {
-	return bf.f.Core().V1().Pods()
-}
-
-func (bf *baseFactory) Services() corev1informers.ServiceInformer {
-	return bf.f.Core().V1().Services()
-}
-
 func (bf *baseFactory) Ingresses() networkingv1informers.IngressInformer {
 	return bf.f.Networking().V1().Ingresses()
 }

internal/informers/core_filteredsecrets.go

@@ -115,14 +115,6 @@ func (bf *filteredSecretsFactory) WaitForCacheSync(stopCh <-chan struct{}) map[s
 	return caches
 }
 
-func (bf *filteredSecretsFactory) Pods() corev1informers.PodInformer {
-	return bf.typedInformerFactory.Core().V1().Pods()
-}
-
-func (bf *filteredSecretsFactory) Services() corev1informers.ServiceInformer {
-	return bf.typedInformerFactory.Core().V1().Services()
-}
-
 func (bf *filteredSecretsFactory) Ingresses() networkingv1informers.IngressInformer {
 	return bf.typedInformerFactory.Networking().V1().Ingresses()
 }

pkg/controller/acmechallenges/controller.go

@@ -94,8 +94,8 @@ func (c *controller) Register(ctx *controllerpkg.Context) (workqueue.RateLimitin
 	secretInformer := ctx.KubeSharedInformerFactory.Secrets()
 	// we register these informers here so the HTTP01 solver has a synced
 	// cache when managing pod/service/ingress resources
-	podInformer := ctx.KubeSharedInformerFactory.Pods()
-	serviceInformer := ctx.KubeSharedInformerFactory.Services()
+	podInformer := ctx.MetadataInformerFactory.ForResource(corev1.SchemeGroupVersion.WithResource("pods"))
+	serviceInformer := ctx.MetadataInformerFactory.ForResource(corev1.SchemeGroupVersion.WithResource("services"))
 	ingressInformer := ctx.KubeSharedInformerFactory.Ingresses()
 
 	// build a list of InformerSynced functions that will be returned by the Register method.

pkg/controller/context.go

@@ -26,11 +26,15 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/metadata"
+	"k8s.io/client-go/metadata/metadatainformer"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
@@ -44,6 +48,7 @@ import (
 	"github.com/cert-manager/cert-manager/internal/controller/feature"
 	internalinformers "github.com/cert-manager/cert-manager/internal/informers"
 	"github.com/cert-manager/cert-manager/pkg/acme/accounts"
+	cmacme "github.com/cert-manager/cert-manager/pkg/apis/acme/v1"
 	clientset "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
 	cmscheme "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/scheme"
 	informers "github.com/cert-manager/cert-manager/pkg/client/informers/externalversions"
@@ -84,6 +89,8 @@ type Context struct {
 	CMClient clientset.Interface
 	// GWClient is a GatewayAPI clientset.
 	GWClient gwclient.Interface
+	// MetadataClient is a PartialObjectMetadata client
+	MetadataClient metadata.Interface
 	// DiscoveryClient is a discovery interface. Usually set to Client.Discovery unless a fake client is in use.
 	DiscoveryClient discovery.DiscoveryInterface
 
@@ -98,6 +105,10 @@ type Context struct {
 	// instances for cert-manager.io types
 	SharedInformerFactory informers.SharedInformerFactory
 
+	// MetadataInformerFactory can be used to start partial metadata
+	// informers
+	MetadataInformerFactory metadatainformer.SharedInformerFactory
+
 	// GWShared can be used to obtain SharedIndexInformer instances for
 	// gateway.networking.k8s.io types
 	GWShared             gwinformers.SharedInformerFactory
@@ -273,6 +284,20 @@ func NewContextFactory(ctx context.Context, opts ContextOptions) (*ContextFactor
 	} else {
 		kubeSharedInformerFactory = internalinformers.NewBaseKubeInformerFactory(clients.kubeClient, resyncPeriod, opts.Namespace)
 	}
+	r, err := labels.NewRequirement(cmacme.DomainLabelKey, selection.Exists, nil)
+	if err != nil {
+		panic(fmt.Errorf("internal error: failed to build label selector to filter HTTP-01 challenge resources: %w", err))
+	}
+	isHTTP01ChallengeResourceLabelSelector := labels.NewSelector().Add(*r)
+	metadataInformerFactory := metadatainformer.NewFilteredSharedInformerFactory(clients.metadataOnlyClient, resyncPeriod, opts.Namespace, func(listOptions *metav1.ListOptions) {
+		// metadataInformersFactory is at the moment only used for pods
+		// and services for http-01 challenge which can be identified by
+		// the same label keys, so it is okay to set the label selector
+		// here. If we start using it for other resources then we'll
+		// have to set the selectors on individual informers instead.
+		listOptions.LabelSelector = isHTTP01ChallengeResourceLabelSelector.String()
+
+	})
 
 	gwSharedInformerFactory := gwinformers.NewSharedInformerFactoryWithOptions(clients.gwClient, resyncPeriod, gwinformers.WithNamespace(opts.Namespace))
 
@@ -286,6 +311,7 @@ func NewContextFactory(ctx context.Context, opts ContextOptions) (*ContextFactor
 			SharedInformerFactory:     sharedInformerFactory,
 			GWShared:                  gwSharedInformerFactory,
 			GatewaySolverEnabled:      clients.gatewayAvailable,
+			MetadataInformerFactory:   metadataInformerFactory,
 			ContextOptions:            opts,
 		},
 	}, nil

pkg/issuer/acme/http/http.go

@@ -29,8 +29,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
-	corev1listers "k8s.io/client-go/listers/core/v1"
 	networkingv1listers "k8s.io/client-go/listers/networking/v1"
+	"k8s.io/client-go/tools/cache"
 	k8snet "k8s.io/utils/net"
 	gwapilisters "sigs.k8s.io/gateway-api/pkg/client/listers/apis/v1beta1"
 
@@ -59,8 +59,8 @@ var (
 type Solver struct {
 	*controller.Context
 
-	podLister       corev1listers.PodLister
-	serviceLister   corev1listers.ServiceLister
+	podLister       cache.GenericLister
+	serviceLister   cache.GenericLister
 	ingressLister   networkingv1listers.IngressLister
 	httpRouteLister gwapilisters.HTTPRouteLister
 
@@ -74,8 +74,8 @@ type reachabilityTest func(ctx context.Context, url *url.URL, key string, dnsSer
 func NewSolver(ctx *controller.Context) (*Solver, error) {
 	return &Solver{
 		Context:          ctx,
-		podLister:        ctx.KubeSharedInformerFactory.Pods().Lister(),
-		serviceLister:    ctx.KubeSharedInformerFactory.Services().Lister(),
+		podLister:        ctx.MetadataInformerFactory.ForResource(corev1.SchemeGroupVersion.WithResource("pods")).Lister(),
+		serviceLister:    ctx.MetadataInformerFactory.ForResource(corev1.SchemeGroupVersion.WithResource("services")).Lister(),
 		ingressLister:    ctx.KubeSharedInformerFactory.Ingresses().Lister(),
 		httpRouteLister:  ctx.GWShared.Gateway().V1beta1().HTTPRoutes().Lister(),
 		testReachability: testReachability,
@@ -108,19 +108,19 @@ func (s *Solver) Present(ctx context.Context, issuer v1.GenericIssuer, ch *cmacm
 	log := logf.FromContext(ctx).WithName(loggerName)
 	ctx = logf.NewContext(ctx, log)
 
-	_, podErr := s.ensurePod(ctx, ch)
-	svc, svcErr := s.ensureService(ctx, ch)
+	podErr := s.ensurePod(ctx, ch)
+	svcName, svcErr := s.ensureService(ctx, ch)
 	if svcErr != nil {
 		return utilerrors.NewAggregate([]error{podErr, svcErr})
 	}
 	var ingressErr, gatewayErr error
 	if ch.Spec.Solver.HTTP01 != nil {
 		if ch.Spec.Solver.HTTP01.Ingress != nil {
-			_, ingressErr = s.ensureIngress(ctx, ch, svc.Name)
+			_, ingressErr = s.ensureIngress(ctx, ch, svcName)
 			return utilerrors.NewAggregate([]error{podErr, svcErr, ingressErr})
 		}
 		if ch.Spec.Solver.HTTP01.GatewayHTTPRoute != nil {
-			_, gatewayErr = s.ensureGatewayHTTPRoute(ctx, ch, svc.Name)
+			_, gatewayErr = s.ensureGatewayHTTPRoute(ctx, ch, svcName)
 			return utilerrors.NewAggregate([]error{podErr, svcErr, gatewayErr})
 		}
 	}

pkg/issuer/acme/http/pod.go

@@ -47,35 +47,36 @@ func podLabels(ch *cmacme.Challenge) map[string]string {
 	}
 }
 
-func (s *Solver) ensurePod(ctx context.Context, ch *cmacme.Challenge) (*corev1.Pod, error) {
+func (s *Solver) ensurePod(ctx context.Context, ch *cmacme.Challenge) error {
 	log := logf.FromContext(ctx).WithName("ensurePod")
 
 	log.V(logf.DebugLevel).Info("checking for existing HTTP01 solver pods")
 	existingPods, err := s.getPodsForChallenge(ctx, ch)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	if len(existingPods) == 1 {
 		logf.WithRelatedResource(log, existingPods[0]).Info("found one existing HTTP01 solver pod")
-		return existingPods[0], nil
+		return nil
 	}
 	if len(existingPods) > 1 {
 		log.V(logf.InfoLevel).Info("multiple challenge solver pods found for challenge. cleaning up all existing pods.")
 		err := s.cleanupPods(ctx, ch)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		return nil, fmt.Errorf("multiple existing challenge solver pods found and cleaned up. retrying challenge sync")
+		return fmt.Errorf("multiple existing challenge solver pods found and cleaned up. retrying challenge sync")
 	}
 
 	log.V(logf.InfoLevel).Info("creating HTTP01 challenge solver pod")
 
-	return s.createPod(ctx, ch)
+	_, err = s.createPod(ctx, ch)
+	return err
 }
 
 // getPodsForChallenge returns a list of pods that were created to solve
 // the given challenge
-func (s *Solver) getPodsForChallenge(ctx context.Context, ch *cmacme.Challenge) ([]*corev1.Pod, error) {
+func (s *Solver) getPodsForChallenge(ctx context.Context, ch *cmacme.Challenge) ([]*metav1.PartialObjectMetadata, error) {
 	log := logf.FromContext(ctx)
 
 	podLabels := podLabels(ch)
@@ -88,19 +89,24 @@ func (s *Solver) getPodsForChallenge(ctx context.Context, ch *cmacme.Challenge)
 		orderSelector = orderSelector.Add(*req)
 	}
 
-	podList, err := s.podLister.Pods(ch.Namespace).List(orderSelector)
+	podMetadataList, err := s.podLister.ByNamespace(ch.Namespace).List(orderSelector)
 	if err != nil {
 		return nil, err
 	}
 
-	var relevantPods []*corev1.Pod
-	for _, pod := range podList {
-		if !metav1.IsControlledBy(pod, ch) {
-			logf.WithRelatedResource(log, pod).Info("found existing solver pod for this challenge resource, however " +
+	var relevantPods []*metav1.PartialObjectMetadata
+	for _, pod := range podMetadataList {
+		// TODO: can we use a metadata lister instead?
+		p, ok := pod.(*metav1.PartialObjectMetadata)
+		if !ok {
+			return nil, fmt.Errorf("internal error: cannot cast PartialMetadata: %+#v", pod)
+		}
+		if !metav1.IsControlledBy(p, ch) {
+			logf.WithRelatedResource(log, p).Info("found existing solver pod for this challenge resource, however " +
 				"it does not have an appropriate OwnerReference referencing this challenge. Skipping it altogether.")
 			continue
 		}
-		relevantPods = append(relevantPods, pod)
+		relevantPods = append(relevantPods, p)
 	}
 
 	return relevantPods, nil

pkg/issuer/acme/http/service.go

@@ -31,34 +31,37 @@ import (
 	logf "github.com/cert-manager/cert-manager/pkg/logs"
 )
 
-func (s *Solver) ensureService(ctx context.Context, ch *cmacme.Challenge) (*corev1.Service, error) {
+// ensureService ensures that a Service exists for the given Challenge. It
+// returns the name of the Service and error if any.
+func (s *Solver) ensureService(ctx context.Context, ch *cmacme.Challenge) (string, error) {
 	log := logf.FromContext(ctx).WithName("ensureService")
 
 	log.V(logf.DebugLevel).Info("checking for existing HTTP01 solver services for challenge")
 	existingServices, err := s.getServicesForChallenge(ctx, ch)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	if len(existingServices) == 1 {
 		logf.WithRelatedResource(log, existingServices[0]).Info("found one existing HTTP01 solver Service for challenge resource")
-		return existingServices[0], nil
+		return existingServices[0].Name, nil
 	}
 	if len(existingServices) > 1 {
 		log.V(logf.DebugLevel).Info("multiple challenge solver services found for challenge. cleaning up all existing services.")
 		err := s.cleanupServices(ctx, ch)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
-		return nil, fmt.Errorf("multiple existing challenge solver services found and cleaned up. retrying challenge sync")
+		return "", fmt.Errorf("multiple existing challenge solver services found and cleaned up. retrying challenge sync")
 	}
 
 	log.V(logf.DebugLevel).Info("creating HTTP01 challenge solver service")
-	return s.createService(ctx, ch)
+	svc, err := s.createService(ctx, ch)
+	return svc.Name, err
 }
 
 // getServicesForChallenge returns a list of services that were created to solve
 // http challenges for the given domain
-func (s *Solver) getServicesForChallenge(ctx context.Context, ch *cmacme.Challenge) ([]*corev1.Service, error) {
+func (s *Solver) getServicesForChallenge(ctx context.Context, ch *cmacme.Challenge) ([]*metav1.PartialObjectMetadata, error) {
 	log := logf.FromContext(ctx).WithName("getServicesForChallenge")
 
 	podLabels := podLabels(ch)
@@ -71,19 +74,24 @@ func (s *Solver) getServicesForChallenge(ctx context.Context, ch *cmacme.Challen
 		selector = selector.Add(*req)
 	}
 
-	serviceList, err := s.serviceLister.Services(ch.Namespace).List(selector)
+	serviceList, err := s.serviceLister.ByNamespace(ch.Namespace).List(selector)
 	if err != nil {
 		return nil, err
 	}
 
-	var relevantServices []*corev1.Service
+	var relevantServices []*metav1.PartialObjectMetadata
 	for _, service := range serviceList {
-		if !metav1.IsControlledBy(service, ch) {
-			logf.WithRelatedResource(log, service).Info("found existing solver pod for this challenge resource, however " +
+		// TODO: can we use a metadata specific lister instead?
+		s, ok := service.(*metav1.PartialObjectMetadata)
+		if !ok {
+			return nil, fmt.Errorf("internal error: cannot cast Service PartialObjectMetadata")
+		}
+		if !metav1.IsControlledBy(s, ch) {
+			logf.WithRelatedResource(log, s).Info("found existing solver pod for this challenge resource, however " +
 				"it does not have an appropriate OwnerReference referencing this challenge. Skipping it altogether.")
 			continue
 		}
-		relevantServices = append(relevantServices, service)
+		relevantServices = append(relevantServices, s)
 	}
 
 	return relevantServices, nil