feat: move cache layer into registry interface

One central place to handle caching

Author: project0

Makefile

@@ -8,7 +8,6 @@ TEST_OPTIONS?=
 export GO111MODULE := on
 export GOPROXY = https://proxy.golang.org,direct
 export GOFLAGS		?=  -tags=containers_image_openpgp,exclude_graphdriver_btrfs,btrfs_noversion,exclude_graphdriver_devicemapper
-export CGO_ENABLED	?=0
 
 help: ## List targets & descriptions
 	@cat Makefile* | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

pkg/backend/cached.go

@@ -41,9 +41,6 @@ func (n *Native) newContext(creds Credentials) *ctypes.SystemContext {
 	return &ctypes.SystemContext{
 		AuthFilePath:     creds.AuthFile,
 		DockerAuthConfig: dockerAuth,
-
-		// It actually defaults to the current runtime, so we may not need to override it
-		// OSChoice: "linux",
 	}
 }
 

pkg/backend/native.go

@@ -65,7 +65,6 @@ func (s *Skopeo) Copy(ctx context.Context, srcRef ctypes.ImageReference, srcCred
 	dest := destRef.DockerReference().String()
 	app := "skopeo"
 	args := []string{
-		"--override-os", "linux",
 		"copy",
 		"--multi-arch", "all",
 		"--retry-times", "3",

pkg/backend/skopeo.go

@@ -0,0 +1,68 @@
+package registry
+
+import (
+	"context"
+
+	ctypes "github.com/containers/image/v5/types"
+	"github.com/dgraph-io/ristretto"
+	"github.com/rs/zerolog/log"
+)
+
+// Cached registry cache requests
+type Cached struct {
+	Cache    *ristretto.Cache
+	Registry Client
+}
+
+func NewCachedClient(cache *ristretto.Cache, registry Client) (*Cached, error) {
+	return &Cached{
+		Registry: registry,
+		Cache:    cache,
+	}, nil
+}
+
+func (c *Cached) CreateRepository(ctx context.Context, name string) error {
+	if _, found := c.Cache.Get(name); found {
+		log.Ctx(ctx).Trace().Str("name", name).Str("method", "CreateRepository").Msg("found in cache")
+		return nil
+	}
+
+	err := c.Registry.CreateRepository(ctx, name)
+
+	if err == nil {
+		c.Cache.Set(name, "", 1)
+	}
+
+	return err
+}
+
+func (c *Cached) ImageExists(ctx context.Context, imageRef ctypes.ImageReference) bool {
+	ref := imageRef.DockerReference().String()
+	if _, found := c.Cache.Get(ref); found {
+		log.Ctx(ctx).Trace().Str("ref", ref).Str("method", "ImageExists").Msg("found in cache")
+		return true
+	}
+
+	exists := c.Registry.ImageExists(ctx, imageRef)
+
+	if exists {
+		c.Cache.Set(ref, "", 1)
+	}
+	return exists
+}
+
+func (c *Cached) CopyImage(ctx context.Context, src ctypes.ImageReference, srcCreds string, dest ctypes.ImageReference, destCreds string) error {
+	return c.Registry.CopyImage(ctx, src, srcCreds, dest, destCreds)
+}
+
+func (c *Cached) Endpoint() string {
+	return c.Registry.Endpoint()
+}
+
+func (c *Cached) Credentials() string {
+	return c.Registry.Credentials()
+}
+
+func (c *Cached) IsOrigin(imageRef ctypes.ImageReference) bool {
+	return c.Registry.IsOrigin(imageRef)
+}

pkg/registry/cache.go

@@ -17,7 +17,6 @@ import (
 // Client provides methods required to be implemented by the various target registry clients, e.g. ECR, Docker, Quay.
 type Client interface {
 	CreateRepository(ctx context.Context, name string) error
-	RepositoryExists() bool
 	CopyImage(ctx context.Context, src ctypes.ImageReference, srcCreds string, dest ctypes.ImageReference, destCreds string) error
 	ImageExists(ctx context.Context, ref ctypes.ImageReference) bool
 
@@ -48,6 +47,7 @@ func NewClient(r config.Registry, imageBackend backend.Backend) (Client, error)
 		return nil, err
 	}
 
+	// TODO: reduce cache size and/or make it configurable
 	cache, err := ristretto.NewCache(&ristretto.Config{
 		NumCounters: 1e7,     // number of keys to track frequency of (10M).
 		MaxCost:     1 << 30, // maximum cost of cache (1GB).
@@ -57,16 +57,21 @@ func NewClient(r config.Registry, imageBackend backend.Backend) (Client, error)
 		return nil, err
 	}
 
-	cachedBackend := backend.NewCached(cache, imageBackend)
-
+	var registryClient Client
 	switch registry {
 	case types.RegistryAWS:
-		return NewECRClient(r.AWS, cachedBackend, cache)
+		if registryClient, err = NewECRClient(r.AWS, imageBackend); err != nil {
+			return nil, err
+		}
 	case types.RegistryGCP:
-		return NewGARClient(r.GCP, cachedBackend)
+		if registryClient, err = NewGARClient(r.GCP, imageBackend); err != nil {
+			return nil, err
+		}
 	default:
 		return nil, fmt.Errorf(`registry of type "%s" is not supported`, r.Type)
 	}
+
+	return NewCachedClient(cache, registryClient)
 }
 
 func GenerateDockerConfig(c Client) ([]byte, error) {

pkg/registry/client.go

@@ -16,7 +16,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/aws/aws-sdk-go/service/ecr/ecriface"
 	ctypes "github.com/containers/image/v5/types"
-	"github.com/dgraph-io/ristretto"
 	"github.com/estahn/k8s-image-swapper/pkg/backend"
 	"github.com/estahn/k8s-image-swapper/pkg/config"
 	"github.com/go-co-op/gocron"
@@ -27,7 +26,6 @@ type ECRClient struct {
 	client          ecriface.ECRAPI
 	ecrDomain       string
 	authToken       []byte
-	cache           *ristretto.Cache
 	scheduler       *gocron.Scheduler
 	targetAccount   string
 	accessPolicy    string
@@ -36,7 +34,7 @@ type ECRClient struct {
 	backend         backend.Backend
 }
 
-func NewECRClient(clientConfig config.AWS, imageBackend backend.Backend, cache *ristretto.Cache) (*ECRClient, error) {
+func NewECRClient(clientConfig config.AWS, imageBackend backend.Backend) (*ECRClient, error) {
 	ecrDomain := clientConfig.EcrDomain()
 
 	var sess *session.Session
@@ -73,7 +71,6 @@ func NewECRClient(clientConfig config.AWS, imageBackend backend.Backend, cache *
 	client := &ECRClient{
 		client:          ecrClient,
 		ecrDomain:       ecrDomain,
-		cache:           cache,
 		scheduler:       scheduler,
 		targetAccount:   clientConfig.AccountID,
 		accessPolicy:    clientConfig.ECROptions.AccessPolicy,
@@ -94,10 +91,6 @@ func (e *ECRClient) Credentials() string {
 }
 
 func (e *ECRClient) CreateRepository(ctx context.Context, name string) error {
-	if _, found := e.cache.Get(name); found {
-		return nil
-	}
-
 	log.Ctx(ctx).Debug().Str("repository", name).Msg("create repository")
 
 	_, err := e.client.CreateRepositoryWithContext(ctx, &ecr.CreateRepositoryInput{
@@ -153,8 +146,6 @@ func (e *ECRClient) CreateRepository(ctx context.Context, name string) error {
 		}
 	}
 
-	e.cache.Set(name, "", 1)
-
 	return nil
 }
 
@@ -189,12 +180,6 @@ func (e *ECRClient) ImageExists(ctx context.Context, imageRef ctypes.ImageRefere
 		Creds: e.Credentials(),
 	}
 
-	ref := imageRef.DockerReference().String()
-	if _, found := e.cache.Get(ref); found {
-		log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in cache")
-		return true
-	}
-
 	exists, err := e.backend.Exists(ctx, imageRef, creds)
 	if err != nil {
 		log.Error().Err(err).Msg("unable to check existence of image")
@@ -266,7 +251,6 @@ func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string
 	client := &ECRClient{
 		client:        ecrClient,
 		ecrDomain:     ecrDomain,
-		cache:         nil,
 		scheduler:     nil,
 		targetAccount: targetAccount,
 		authToken:     []byte("mock-ecr-client-fake-auth-token"),