From be3c9e5da64693a1bb95a8e576c8412f53044458 Mon Sep 17 00:00:00 2001 From: John Mehan Date: Mon, 23 Oct 2023 10:10:37 -0400 Subject: [PATCH 1/3] feat: addition of generic registry --- .gitignore | 3 + cmd/root.go | 6 + docs/configuration.md | 39 +++++ pkg/config/config.go | 76 ++++++--- pkg/config/config_test.go | 256 ++++++++++++++++++++++++++++++ pkg/registry/client.go | 8 +- pkg/registry/client_test.go | 84 ++++++++++ pkg/registry/cmd-executor.go | 22 +++ pkg/registry/cmd-executor_test.go | 50 ++++++ pkg/registry/ecr.go | 50 +++--- pkg/registry/ecr_test.go | 208 +++++++++++++++++++++++- pkg/registry/gar.go | 30 ++-- pkg/registry/gar_test.go | 228 +++++++++++++++++++++++++- pkg/registry/generic.go | 209 ++++++++++++++++++++++++ pkg/registry/generic_test.go | 256 ++++++++++++++++++++++++++++++ pkg/registry/inmemory.go | 1 - pkg/secrets/dummy.go | 1 + pkg/types/types.go | 5 +- pkg/types/types_test.go | 34 +++- pkg/webhook/image_copier.go | 28 +++- pkg/webhook/image_copier_test.go | 8 +- pkg/webhook/image_swapper.go | 181 ++++++++++++--------- pkg/webhook/image_swapper_test.go | 18 ++- 23 files changed, 1641 insertions(+), 160 deletions(-) create mode 100644 pkg/registry/client_test.go create mode 100644 pkg/registry/cmd-executor.go create mode 100644 pkg/registry/cmd-executor_test.go create mode 100644 pkg/registry/generic.go create mode 100644 pkg/registry/generic_test.go delete mode 100644 pkg/registry/inmemory.go diff --git a/.gitignore b/.gitignore index cead1d23..c4403711 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ # Dependency directories (remove the comment below to include it) # vendor/ +# Mac +.DS_Store + .idea/ coverage.txt k8s-image-swapper diff --git a/cmd/root.go b/cmd/root.go index f10a3999..bc5074ed 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -66,20 +66,25 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`, // Create registry clients for source registries sourceRegistryClients := []registry.Client{} for _, reg := range cfg.Source.Registries { + log.Trace().Msgf("Connecting to Source Registry") sourceRegistryClient, err := registry.NewClient(reg) if err != nil { log.Err(err).Msgf("error connecting to source registry at %s", reg.Domain()) os.Exit(1) } + log.Trace().Msgf("Added Source Registry: %s", sourceRegistryClient.Endpoint()) sourceRegistryClients = append(sourceRegistryClients, sourceRegistryClient) } // Create a registry client for private target registry + + log.Trace().Msgf("Connecting to Target Registry") targetRegistryClient, err := registry.NewClient(cfg.Target) if err != nil { log.Err(err).Msgf("error connecting to target registry at %s", cfg.Target.Domain()) os.Exit(1) } + log.Trace().Msgf("Added Target Registry: %s", targetRegistryClient.Endpoint()) imageSwapPolicy, err := types.ParseImageSwapPolicy(cfg.ImageSwapPolicy) if err != nil { @@ -102,6 +107,7 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`, imagePullSecretProvider.SetAuthenticatedRegistries(sourceRegistryClients) wh, err := webhook.NewImageSwapperWebhookWithOpts( + sourceRegistryClients, targetRegistryClient, webhook.Filters(cfg.Source.Filters), webhook.ImagePullSecretsProvider(imagePullSecretProvider), diff --git a/docs/configuration.md b/docs/configuration.md index 54f6b210..81a77fed 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -77,6 +77,30 @@ Registries are described with an AWS account ID and region, mostly to construct accountId: 234567890 region: us-east-1 ``` +#### GENERIC + +By providing configuration on GENERIC registries you can ask `k8s-image-swapper` to handle the authentication using +username and password. + +Registries are described with a repository URL, username and password. + +!!! example + ```yaml + source: + registries: + - type: "generic" + generic: + repository: "repo1.azurecr.io" + username: "username1" + password: "pass1" + - type: "generic" + generic: + repository: "repo2.azurecr.io" + username: "username2" + password: "pass2" + ``` + + ### Filters Filters provide control over what pods will be processed. @@ -172,6 +196,7 @@ The AWS Account ID and Region is primarily used to construct the ECR domain `[AC region: ap-southeast-2 ``` + #### ECR Options ##### Tags @@ -204,3 +229,17 @@ The GCP location, projectId, and repositoryId are used to constrct the GCP Artif projectId: gcp-project-123 repositoryId: main ``` + +### GENERIC + +The option `target.generic` holds details about the target registry storing the images. + +!!! example + ```yaml + target: + type: generic + generic: + repository: "repo2.azurecr.io" + username: "username2" + password: "pass2" + ``` diff --git a/pkg/config/config.go b/pkg/config/config.go index c331d395..20e29bed 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -57,9 +57,10 @@ type Source struct { } type Registry struct { - Type string `yaml:"type"` - AWS AWS `yaml:"aws"` - GCP GCP `yaml:"gcp"` + Type string `yaml:"type"` + GENERIC GENERIC `yaml:"generic"` + AWS AWS `yaml:"aws"` + GCP GCP `yaml:"gcp"` } type AWS struct { @@ -75,6 +76,13 @@ type GCP struct { RepositoryID string `yaml:"repositoryId"` } +type GENERIC struct { + Repository string `yaml:"repository"` + Username string `yaml:"username"` + Password string `yaml:"password"` + IgnoreCert bool `yaml:"ignoreCert"` +} + type ECROptions struct { AccessPolicy string `yaml:"accessPolicy"` LifecyclePolicy string `yaml:"lifecyclePolicy"` @@ -124,30 +132,54 @@ func CheckRegistryConfiguration(r Registry) error { return fmt.Errorf("a registry requires a type") } - errorWithType := func(info string) error { - return fmt.Errorf(`registry of type "%s" %s`, r.Type, info) - } - registry, _ := types.ParseRegistry(r.Type) switch registry { case types.RegistryAWS: - if r.AWS.Region == "" { - return errorWithType(`requires a field "region"`) - } - if r.AWS.AccountID == "" { - return errorWithType(`requires a field "accountdId"`) - } + return validateAWSRegistry(r) case types.RegistryGCP: - if r.GCP.Location == "" { - return errorWithType(`requires a field "location"`) - } - if r.GCP.ProjectID == "" { - return errorWithType(`requires a field "projectId"`) - } - if r.GCP.RepositoryID == "" { - return errorWithType(`requires a field "repositoryId"`) - } + return validateGCPRegistry(r) + case types.RegistryGeneric: + return validateGenericRegistry(r) + } + + return nil +} + +func errorWithType(r Registry, info string) error { + return fmt.Errorf(`registry of type "%s" %s`, r.Type, info) +} +func validateAWSRegistry(r Registry) error { + if r.AWS.Region == "" { + return errorWithType(r, "requires a field region") + } + if r.AWS.AccountID == "" { + return errorWithType(r, `requires a field "accountdId"`) } + return nil +} +func validateGCPRegistry(r Registry) error { + if r.GCP.Location == "" { + return errorWithType(r, `requires a field "location"`) + } + if r.GCP.ProjectID == "" { + return errorWithType(r, `requires a field "projectId"`) + } + if r.GCP.RepositoryID == "" { + return errorWithType(r, `requires a field "repositoryId"`) + } + return nil +} + +func validateGenericRegistry(r Registry) error { + if r.GENERIC.Repository == "" { + return errorWithType(r, `requires a field "repository"`) + } + if r.GENERIC.Username == "" { + return errorWithType(r, `requires a field "username"`) + } + if r.GENERIC.Password == "" { + return errorWithType(r, `requires a field "password"`) + } return nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 631ed6e4..fd88c676 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -8,6 +8,40 @@ import ( ) // TestConfigParses validates if yaml annotation do not overlap +const defaultConfig = ` +source: + registries: + - type: "aws" + aws: + accountId: "12345678912" + region: "us-west-1" + - type: "generic" + generic: + repository: "https://12345678912" + username: "demo" + password: "pass" + - type: "gcp" + gcp: + location: "us-east" + projectId: "12345" + repositoryId: "67890" + filters: + - jmespath: "obj.metadata.namespace == 'kube-system'" + - jmespath: "obj.metadata.namespace != 'playground'" +target: + type: aws + aws: + accountId: 123456789 + region: ap-southeast-2 + role: arn:aws:iam::123456789012:role/roleName + ecrOptions: + tags: + - key: CreatedBy + value: k8s-image-swapper + - key: A + value: B +` + func TestConfigParses(t *testing.T) { tests := []struct { name string @@ -109,6 +143,51 @@ source: }, }, }, + { + name: "should render multiple source registries", + cfg: ` +source: + registries: + - type: "aws" + aws: + accountId: "12345678912" + region: "us-west-1" + - type: "generic" + generic: + repository: "https://12345678912" + username: "demo" + password: "pass" + - type: "aws" + aws: + accountId: "12345678912" + region: "us-east-1" +`, + expCfg: Config{ + Source: Source{ + Registries: []Registry{ + { + Type: "aws", + AWS: AWS{ + AccountID: "12345678912", + Region: "us-west-1", + }}, + { + Type: "generic", + GENERIC: GENERIC{ + Repository: "https://12345678912", + Username: "demo", + Password: "pass", + }}, + { + Type: "aws", + AWS: AWS{ + AccountID: "12345678912", + Region: "us-east-1", + }}, + }, + }, + }, + }, } for _, test := range tests { @@ -126,3 +205,180 @@ source: }) } } + +func TestSuccess(t *testing.T) { + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + awsRegistry := cfg.Source.Registries[0] + awsDomain := awsRegistry.Domain() + assert.Equal(t, "12345678912.dkr.ecr.us-west-1.amazonaws.com", awsDomain) + assert.Nil(t, CheckRegistryConfiguration(awsRegistry)) + + genericRegistry := cfg.Source.Registries[1] + genericDomain := genericRegistry.Domain() + assert.Equal(t, "", genericDomain) + assert.Nil(t, CheckRegistryConfiguration(genericRegistry)) + + gcpRegistry := cfg.Source.Registries[2] + gcpDomain := gcpRegistry.Domain() + assert.Equal(t, "us-east-docker.pkg.dev/12345/67890", gcpDomain) + assert.Nil(t, CheckRegistryConfiguration(gcpRegistry)) +} + +func TestNoRegistryType(t *testing.T) { + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + registry := cfg.Source.Registries[0] + registry.Type = "" + + err = CheckRegistryConfiguration(registry) + assert.NotNil(t, err) + assert.Equal(t, "a registry requires a type", err.Error()) +} + +func TestUnknownRegistryType(t *testing.T) { + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + registry := cfg.Source.Registries[0] + registry.Type = "TEST" + + err = CheckRegistryConfiguration(registry) + assert.Nil(t, err) +} + +func TestAWSRegistryNoRegion(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + awsRegistry := cfg.Source.Registries[0] + awsRegistry.AWS.Region = "" + + err = CheckRegistryConfiguration(awsRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"aws\" requires a field region", err.Error()) + +} + +func TestAWSRegistryNoAccount(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + awsRegistry := cfg.Source.Registries[0] + awsRegistry.AWS.AccountID = "" + + err = CheckRegistryConfiguration(awsRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"aws\" requires a field \"accountdId\"", err.Error()) + +} + +func TestGenericRegistryNoRepository(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + genericRegistry := cfg.Source.Registries[1] + genericRegistry.GENERIC.Repository = "" + + err = CheckRegistryConfiguration(genericRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"generic\" requires a field \"repository\"", err.Error()) + +} + +func TestGenericRegistryNoUsername(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + genericRegistry := cfg.Source.Registries[1] + genericRegistry.GENERIC.Username = "" + + err = CheckRegistryConfiguration(genericRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"generic\" requires a field \"username\"", err.Error()) + +} + +func TestGenericRegistryNoPassword(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + genericRegistry := cfg.Source.Registries[1] + genericRegistry.GENERIC.Password = "" + + err = CheckRegistryConfiguration(genericRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"generic\" requires a field \"password\"", err.Error()) + +} + +func TestGCPRegistryNoLocation(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + gcpRegistry := cfg.Source.Registries[2] + gcpRegistry.GCP.Location = "" + + err = CheckRegistryConfiguration(gcpRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"gcp\" requires a field \"location\"", err.Error()) + +} + +func TestGCPRegistryNoRepositoryID(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + gcpRegistry := cfg.Source.Registries[2] + gcpRegistry.GCP.RepositoryID = "" + + err = CheckRegistryConfiguration(gcpRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"gcp\" requires a field \"repositoryId\"", err.Error()) + +} + +func TestGCPRegistryNoProjectID(t *testing.T) { + + cfg := Config{} + err := yaml.Unmarshal([]byte(defaultConfig), &cfg) + assert.Nil(t, err) + assert.NotNil(t, cfg) + + gcpRegistry := cfg.Source.Registries[2] + gcpRegistry.GCP.ProjectID = "" + + err = CheckRegistryConfiguration(gcpRegistry) + assert.NotNil(t, err) + assert.Equal(t, "registry of type \"gcp\" requires a field \"projectId\"", err.Error()) + +} diff --git a/pkg/registry/client.go b/pkg/registry/client.go index da373e2c..db9f432d 100644 --- a/pkg/registry/client.go +++ b/pkg/registry/client.go @@ -12,13 +12,13 @@ import ( ctypes "github.com/containers/image/v5/types" ) +const implementMe = "implement me" +const dockerPrefix = "docker://" + // Client provides methods required to be implemented by the various target registry clients, e.g. ECR, Docker, Quay. type Client interface { CreateRepository(ctx context.Context, name string) error - RepositoryExists() bool CopyImage(ctx context.Context, src ctypes.ImageReference, srcCreds string, dest ctypes.ImageReference, destCreds string) error - PullImage() error - PutImage() error ImageExists(ctx context.Context, ref ctypes.ImageReference) bool // Endpoint returns the domain of the registry @@ -53,6 +53,8 @@ func NewClient(r config.Registry) (Client, error) { return NewECRClient(r.AWS) case types.RegistryGCP: return NewGARClient(r.GCP) + case types.RegistryGeneric: + return NewGenericClient(r.GENERIC) default: return nil, fmt.Errorf(`registry of type "%s" is not supported`, r.Type) } diff --git a/pkg/registry/client_test.go b/pkg/registry/client_test.go new file mode 100644 index 00000000..2fff5893 --- /dev/null +++ b/pkg/registry/client_test.go @@ -0,0 +1,84 @@ +package registry + +import ( + "context" + "fmt" + "testing" + + "github.com/estahn/k8s-image-swapper/pkg/config" + "github.com/stretchr/testify/assert" +) + +var genConfig = config.GENERIC{ + Repository: "localhost", + Username: "user", + Password: "password", + IgnoreCert: true, +} + +func TestNewClientSuccess(t *testing.T) { + + genConfig = config.GENERIC{ + Repository: "localhost", + Username: "user", + Password: "password", + IgnoreCert: true, + } + r := config.Registry{ + Type: "generic", + GENERIC: genConfig, + } + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte("login successful"), + err: nil, + } + } + + client, err := NewClient(r) + assert.Nil(t, err) + assert.NotNil(t, client) +} + +func TestNewClientFailureNoType(t *testing.T) { + + genConfig = config.GENERIC{ + Repository: "localhost", + Username: "user", + Password: "password", + IgnoreCert: true, + } + r := config.Registry{ + Type: "", + GENERIC: genConfig, + } + + client, err := NewClient(r) + assert.NotNil(t, err) + assert.Nil(t, client) + assert.Equal(t, "a registry requires a type", err.Error()) +} + +func TestNewClientFailureInvalidType(t *testing.T) { + + genConfig = config.GENERIC{ + Repository: "localhost", + Username: "user", + Password: "password", + IgnoreCert: true, + } + r := config.Registry{ + Type: "badType", + GENERIC: genConfig, + } + + client, err := NewClient(r) + assert.NotNil(t, err) + assert.Nil(t, client) + assert.Equal(t, "unknown target registry string: 'badType', defaulting to unknown", err.Error()) +} diff --git a/pkg/registry/cmd-executor.go b/pkg/registry/cmd-executor.go new file mode 100644 index 00000000..15238b5f --- /dev/null +++ b/pkg/registry/cmd-executor.go @@ -0,0 +1,22 @@ +package registry + +import ( + "context" + "os/exec" +) + +type ShellCommand interface { + CombinedOutput() ([]byte, error) + Run() error +} + +type execShellCommand struct { + *exec.Cmd +} + +func newCommandExecutor(ctx context.Context, name string, arg ...string) ShellCommand { + execCmd := exec.CommandContext(ctx, name, arg...) + return execShellCommand{Cmd: execCmd} +} + +var commandExecutor = newCommandExecutor diff --git a/pkg/registry/cmd-executor_test.go b/pkg/registry/cmd-executor_test.go new file mode 100644 index 00000000..9161eff9 --- /dev/null +++ b/pkg/registry/cmd-executor_test.go @@ -0,0 +1,50 @@ +package registry + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +type testCommandExecutor struct { + CombinedOutputFunc func() ([]byte, error) + output []byte + err error +} + +func (tsc testCommandExecutor) CombinedOutput() ([]byte, error) { + return tsc.output, tsc.err +} + +func (tsc testCommandExecutor) Run() error { + return tsc.err +} + +func TestSuccess(t *testing.T) { + + ctx := context.Background() + app := "app" + args := "args" + + shellCmd := commandExecutor(ctx, app, args) + assert.NotNil(t, shellCmd) + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte("command not found"), + err: errors.New("copy error"), + } + } + + newShellCmd := commandExecutor(ctx, app, args) + assert.NotNil(t, newShellCmd) + + output, cmdErr := newShellCmd.CombinedOutput() + assert.Equal(t, "command not found", string(output)) + assert.NotNil(t, cmdErr) + assert.Equal(t, "copy error", cmdErr.Error()) +} diff --git a/pkg/registry/ecr.go b/pkg/registry/ecr.go index f273812f..f9dc2fc0 100644 --- a/pkg/registry/ecr.go +++ b/pkg/registry/ecr.go @@ -4,11 +4,11 @@ import ( "context" "encoding/base64" "fmt" - "github.com/containers/image/v5/docker/reference" "net/http" - "os/exec" "time" + "github.com/containers/image/v5/docker/reference" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" @@ -35,6 +35,14 @@ type ECRClient struct { } func NewECRClient(clientConfig config.AWS) (*ECRClient, error) { + + client := initClient(clientConfig) + if err := client.scheduleTokenRenewal(); err != nil { + return nil, err + } + return client, nil +} +func initClient(clientConfig config.AWS) *ECRClient { ecrDomain := clientConfig.EcrDomain() var sess *session.Session @@ -87,12 +95,7 @@ func NewECRClient(clientConfig config.AWS) (*ECRClient, error) { lifecyclePolicy: clientConfig.ECROptions.LifecyclePolicy, tags: clientConfig.ECROptions.Tags, } - - if err := client.scheduleTokenRenewal(); err != nil { - return nil, err - } - - return client, nil + return client } func (e *ECRClient) Credentials() string { @@ -175,11 +178,8 @@ func (e *ECRClient) buildEcrTags() []*ecr.Tag { return ecrTags } -func (e *ECRClient) RepositoryExists() bool { - panic("implement me") -} - func (e *ECRClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, srcCreds string, destRef ctypes.ImageReference, destCreds string) error { + src := srcRef.DockerReference().String() dest := destRef.DockerReference().String() app := "skopeo" @@ -188,8 +188,8 @@ func (e *ECRClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, "copy", "--multi-arch", "all", "--retry-times", "3", - "docker://" + src, - "docker://" + dest, + dockerPrefix + src, + dockerPrefix + dest, } if len(srcCreds) > 0 { @@ -210,7 +210,7 @@ func (e *ECRClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, Strs("args", args). Msg("execute command to copy image") - output, cmdErr := exec.CommandContext(ctx, app, args...).CombinedOutput() + output, cmdErr := commandExecutor(ctx, app, args...).CombinedOutput() // check if the command timed out during execution for proper logging if err := ctx.Err(); err != nil { @@ -225,14 +225,6 @@ func (e *ECRClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, return nil } -func (e *ECRClient) PullImage() error { - panic("implement me") -} - -func (e *ECRClient) PutImage() error { - panic("implement me") -} - func (e *ECRClient) ImageExists(ctx context.Context, imageRef ctypes.ImageReference) bool { ref := imageRef.DockerReference().String() if _, found := e.cache.Get(ref); found { @@ -244,12 +236,12 @@ func (e *ECRClient) ImageExists(ctx context.Context, imageRef ctypes.ImageRefere args := []string{ "inspect", "--retry-times", "3", - "docker://" + ref, + dockerPrefix + ref, "--creds", e.Credentials(), } log.Ctx(ctx).Trace().Str("app", app).Strs("args", args).Msg("executing command to inspect image") - if err := exec.CommandContext(ctx, app, args...).Run(); err != nil { + if err := commandExecutor(ctx, app, args...).Run(); err != nil { log.Ctx(ctx).Trace().Str("ref", ref).Msg("not found in target repository") return false } @@ -309,12 +301,20 @@ func (e *ECRClient) scheduleTokenRenewal() error { // For testing purposes func NewDummyECRClient(region string, targetAccount string, role string, options config.ECROptions, authToken []byte) *ECRClient { + + cache, _ := ristretto.NewCache(&ristretto.Config{ + NumCounters: 10, // number of keys to track frequency of (10M). + MaxCost: 1 << 30, // maximum cost of cache (1GB). + BufferItems: 1, // number of keys per Get buffer. + }) + return &ECRClient{ targetAccount: targetAccount, accessPolicy: options.AccessPolicy, lifecyclePolicy: options.LifecyclePolicy, ecrDomain: fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com", targetAccount, region), authToken: authToken, + cache: cache, } } diff --git a/pkg/registry/ecr_test.go b/pkg/registry/ecr_test.go index 54757b2e..a71f0d1f 100644 --- a/pkg/registry/ecr_test.go +++ b/pkg/registry/ecr_test.go @@ -1,10 +1,14 @@ package registry import ( + "context" "encoding/base64" - "github.com/containers/image/v5/transports/alltransports" + "errors" + "fmt" "testing" + "time" + "github.com/containers/image/v5/transports/alltransports" "github.com/estahn/k8s-image-swapper/pkg/config" "github.com/stretchr/testify/assert" ) @@ -50,3 +54,205 @@ func TestECRIsOrigin(t *testing.T) { assert.Equal(t, testcase.expected, result) } } + +func TestECRClientCopyImageSuccess(t *testing.T) { + + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + srcRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "user:pass" + destCreds := "user:pass" + + err := ecrClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.Nil(t, err) + +} + +func TestECRClientCopyImageSuccessNoCreds(t *testing.T) { + + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + srcRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "" + destCreds := "" + + err := ecrClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.Nil(t, err) + +} + +func TestECRClientCopyImageFailure(t *testing.T) { + + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte("missing"), + err: errors.New("Command Failed"), + } + } + + srcRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "" + destCreds := "" + + err := ecrClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.NotNil(t, err) + assert.Equal(t, "Command error, stderr: Command Failed, stdout: missing", err.Error()) + +} + +func TestECRClientIsNotOrigin(t *testing.T) { + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + imageRef, _ := alltransports.ParseImageName("docker://test-ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + isOrigin := ecrClient.IsOrigin(imageRef) + assert.False(t, isOrigin) +} + +func TestECRClientIsOrigin(t *testing.T) { + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + imageRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + isOrigin := ecrClient.IsOrigin(imageRef) + assert.True(t, isOrigin) +} + +func TestECRClientImageExistsSuccess(t *testing.T) { + + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + imageRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + exists := ecrClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, true, exists) +} + +func TestECRClientImageExistsInCacheSuccess(t *testing.T) { + + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + imageRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + ecrClient.cache.Set(imageRef.DockerReference().String(), "123", 123) + time.Sleep(time.Millisecond * 10) + + exists := ecrClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, true, exists) +} + +func TestECRClientImageDoesNotExistsSuccess(t *testing.T) { + + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + err: errors.New("Image not found"), + } + } + + imageRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + exists := ecrClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, false, exists) +} + +func TestInitClientSuccess(t *testing.T) { + + config := config.AWS{ + AccountID: "123", + Region: "us-east-1", + Role: "", + ECROptions: config.ECROptions{}, + } + + client := initClient(config) + + assert.NotNil(t, client) + assert.Equal(t, "123.dkr.ecr.us-east-1.amazonaws.com", client.Endpoint()) + assert.Equal(t, "123", client.targetAccount) +} +func TestInitClientWithRoleSuccess(t *testing.T) { + + config := config.AWS{ + AccountID: "123", + Region: "us-east-1", + Role: "admin", + ECROptions: config.ECROptions{}, + } + + client := initClient(config) + + assert.NotNil(t, client) + assert.Equal(t, "123.dkr.ecr.us-east-1.amazonaws.com", client.Endpoint()) + assert.Equal(t, "123", client.targetAccount) +} + +func TestCreateRepositoryInCache(t *testing.T) { + ecrClient := NewDummyECRClient("us-east-1", "12345678912", "", config.ECROptions{}, []byte("")) + ecrClient.cache.Set("reg1", "123", 1) + time.Sleep(time.Millisecond * 10) + err := ecrClient.CreateRepository(context.Background(), "reg1") + assert.Nil(t, err) + +} + +func TestBuildEcrTagsSuccess(t *testing.T) { + registryClient, _ := NewMockECRClient(nil, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") + assert.NotNil(t, registryClient) + ecrTags := registryClient.buildEcrTags() + assert.NotNil(t, ecrTags) + assert.Len(t, ecrTags, 2) + assert.Equal(t, "CreatedBy", *ecrTags[0].Key) + assert.Equal(t, "k8s-image-swapper", *ecrTags[0].Value) + assert.Equal(t, "AnotherTag", *ecrTags[1].Key) + assert.Equal(t, "another-tag", *ecrTags[1].Value) +} diff --git a/pkg/registry/gar.go b/pkg/registry/gar.go index f5898b48..664e3de9 100644 --- a/pkg/registry/gar.go +++ b/pkg/registry/gar.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "os/exec" "strings" "time" @@ -63,10 +62,6 @@ func (e *GARClient) CreateRepository(ctx context.Context, name string) error { return nil } -func (e *GARClient) RepositoryExists() bool { - panic("implement me") -} - func (e *GARClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, srcCreds string, destRef ctypes.ImageReference, destCreds string) error { src := srcRef.DockerReference().String() dest := destRef.DockerReference().String() @@ -84,8 +79,8 @@ func (e *GARClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, "copy", "--multi-arch", "all", "--retry-times", "3", - "docker://" + src, - "docker://" + dest, + dockerPrefix + src, + dockerPrefix + dest, } if len(creds[1]) > 0 { @@ -106,7 +101,7 @@ func (e *GARClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, Strs("args", args). Msg("execute command to copy image") - output, cmdErr := exec.CommandContext(ctx, app, args...).CombinedOutput() + output, cmdErr := commandExecutor(ctx, app, args...).CombinedOutput() // check if the command timed out during execution for proper logging if err := ctx.Err(); err != nil { @@ -121,14 +116,6 @@ func (e *GARClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, return nil } -func (e *GARClient) PullImage() error { - panic("implement me") -} - -func (e *GARClient) PutImage() error { - panic("implement me") -} - func (e *GARClient) ImageExists(ctx context.Context, imageRef ctypes.ImageReference) bool { ref := imageRef.DockerReference().String() if _, found := e.cache.Get(ref); found { @@ -140,12 +127,12 @@ func (e *GARClient) ImageExists(ctx context.Context, imageRef ctypes.ImageRefere args := []string{ "inspect", "--retry-times", "3", - "docker://" + ref, + dockerPrefix + ref, "--creds", e.Credentials(), } log.Ctx(ctx).Trace().Str("app", app).Strs("args", args).Msg("executing command to inspect image") - if err := exec.CommandContext(ctx, app, args...).Run(); err != nil { + if err := commandExecutor(ctx, app, args...).Run(); err != nil { log.Trace().Str("ref", ref).Msg("not found in target repository") return false } @@ -231,5 +218,12 @@ func NewMockGARClient(garClient GARAPI, garDomain string) (*GARClient, error) { authToken: []byte("oauth2accesstoken:mock-gar-client-fake-auth-token"), } + cache, _ := ristretto.NewCache(&ristretto.Config{ + NumCounters: 10, // number of keys to track frequency of (10M). + MaxCost: 1 << 30, // maximum cost of cache (1GB). + BufferItems: 1, // number of keys per Get buffer. + }) + client.cache = cache + return client, nil } diff --git a/pkg/registry/gar_test.go b/pkg/registry/gar_test.go index 5e9c8b62..785f7f28 100644 --- a/pkg/registry/gar_test.go +++ b/pkg/registry/gar_test.go @@ -1,9 +1,16 @@ package registry import ( - "github.com/containers/image/v5/transports/alltransports" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" "testing" + "time" + "github.com/containers/image/v5/transports/alltransports" + "github.com/estahn/k8s-image-swapper/pkg/config" "github.com/stretchr/testify/assert" ) @@ -35,3 +42,222 @@ func TestGARIsOrigin(t *testing.T) { assert.Equal(t, testcase.expected, result) } } + +func TestGARClientCopyImageSuccess(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + srcRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "user:pass" + destCreds := "user:pass" + + err := garClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.Nil(t, err) + +} + +func TestGARClientCopyImageWithSuffixSuccess(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + srcRef, _ := alltransports.ParseImageName("docker://12345678912-docker.pkg.dev/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "user:pass" + destCreds := "user:pass" + + err := garClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.Nil(t, err) + +} + +func TestGARClientCopyImageSuccessNoCreds(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + srcRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "" + destCreds := "" + + err := garClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.Nil(t, err) + +} + +func TestGARClientCopyImageFailure(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte("missing"), + err: errors.New("Command Failed"), + } + } + + srcRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "" + destCreds := "" + + err := garClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.NotNil(t, err) + assert.Equal(t, "Command error, stderr: Command Failed, stdout: missing", err.Error()) + +} + +func TestGARClientIsNotOrigin(t *testing.T) { + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + imageRef, _ := alltransports.ParseImageName("docker://test-ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + isOrigin := garClient.IsOrigin(imageRef) + assert.False(t, isOrigin) +} + +func TestGARClientIsOrigin(t *testing.T) { + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + imageRef, _ := alltransports.ParseImageName("docker://us-central1-docker.pkg.dev/gcp-project-123/main/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + isOrigin := garClient.IsOrigin(imageRef) + assert.True(t, isOrigin) +} + +func TestGARClientImageExistsSuccess(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + imageRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + exists := garClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, true, exists) +} + +func TestGARClientImageExistsInCacheSuccess(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + imageRef, _ := alltransports.ParseImageName("docker://us-central1-docker.pkg.dev/gcp-project-123/main/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + garClient.cache.Set(imageRef.DockerReference().String(), "123", 123) + time.Sleep(time.Millisecond * 10) + + exists := garClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, true, exists) +} + +func TestGARClientImageDoesNotExistsSuccess(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + err: errors.New("Image not found"), + } + } + + imageRef, _ := alltransports.ParseImageName("docker://12345678912.dkr.ecr.us-east-1.amazonaws.com/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + exists := garClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, false, exists) +} + +func TestGARClientDockerConfigSuccess(t *testing.T) { + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + + data, err := garClient.DockerConfig() + assert.Nil(t, err) + assert.NotNil(t, data) + + dockerConfig := &DockerConfig{} + err = json.Unmarshal(data, dockerConfig) + assert.Nil(t, err) + + for key, authConfig := range dockerConfig.AuthConfigs { + assert.Equal(t, "us-central1-docker.pkg.dev/gcp-project-123/main", key) + assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("oauth2accesstoken:mock-gar-client-fake-auth-token")), authConfig.Auth) + } +} +func TestGARClientCreateRegistryNilResponse(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + err := garClient.CreateRepository(context.Background(), "repo") + assert.Nil(t, err) +} + +func TestGARClientScheduleTokenRenewalFailure(t *testing.T) { + + garClient, _ := NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + err := garClient.scheduleTokenRenewal() + assert.NotNil(t, err) +} + +func TestGARClientNewClientFailure(t *testing.T) { + + cfg := config.GCP{ + Location: "us-central1-docker.pkg.dev/gcp-project-123/main", + ProjectID: "123", + RepositoryID: "456", + } + + garClient, err := NewGARClient(cfg) + assert.NotNil(t, err) + assert.Nil(t, garClient) +} diff --git a/pkg/registry/generic.go b/pkg/registry/generic.go new file mode 100644 index 00000000..40c671e1 --- /dev/null +++ b/pkg/registry/generic.go @@ -0,0 +1,209 @@ +package registry + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + ctypes "github.com/containers/image/v5/types" + "github.com/dgraph-io/ristretto" + "github.com/estahn/k8s-image-swapper/pkg/config" + "github.com/rs/zerolog/log" +) + +type GenericAPI interface{} + +type GenericClient struct { + client GenericAPI + repository string + username string + password string + ignoreCert bool + cache *ristretto.Cache +} + +func NewGenericClient(clientConfig config.GENERIC) (*GenericClient, error) { + + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, // number of keys to track frequency of (10M). + MaxCost: 1 << 30, // maximum cost of cache (1GB). + BufferItems: 64, // number of keys per Get buffer. + }) + if err != nil { + return nil, err + } + + var genericClient = &GenericClient{ + repository: clientConfig.Repository, + username: clientConfig.Username, + password: clientConfig.Password, + ignoreCert: clientConfig.IgnoreCert, + cache: cache, + } + + err = genericClient.login() + if err != nil { + return nil, err + } + + return genericClient, nil +} + +func (g *GenericClient) login() error { + + ctx := context.Background() + app := "skopeo" + args := []string{ + "login", + "-u", g.username, + "--password", g.password, + g.repository, + } + + if g.ignoreCert { + args = append(args, "--tls-verify=false") + } + + log.Ctx(ctx). + Trace(). + Str("app", app). + Strs("args", args). + Msg("execute command to login to repository") + + log.Trace().Msgf("GenericClient:login - app args %v", args) + + command := commandExecutor(ctx, app, args...) + output, cmdErr := command.CombinedOutput() + + // enrich error with output from the command which may contain the actual reason + if cmdErr != nil { + log.Trace().Msgf("GenericClient:login - Command error, stderr: %s, stdout: %s", cmdErr.Error(), string(output)) + return fmt.Errorf("Command error, stderr: %s, stdout: %s", cmdErr.Error(), string(output)) + } + + return nil +} + +func (g *GenericClient) CopyImage(ctx context.Context, srcRef ctypes.ImageReference, srcCreds string, destRef ctypes.ImageReference, destCreds string) error { + src := srcRef.DockerReference().String() + dest := destRef.DockerReference().String() + + app := "skopeo" + args := []string{ + "--override-os", "linux", + "copy", + "--multi-arch", "all", + "--retry-times", "3", + dockerPrefix + src, + dockerPrefix + dest, + } + + //ignore both certs if destination cert is ignored + if g.ignoreCert { + args = append(args, "--src-tls-verify=false") + args = append(args, "--dest-tls-verify=false") + } + + if len(srcCreds) > 0 { + args = append(args, "--src-authfile", srcCreds) + } else { + args = append(args, "--src-no-creds") + } + + if len(destCreds) > 0 { + args = append(args, "--dest-creds", destCreds) + } else { + args = append(args, "--dest-no-creds") + } + + log.Ctx(ctx). + Trace(). + Str("app", app). + Strs("args", args). + Msg("execute command to copy image") + + log.Trace().Msgf("GenericClient:CopyImage - app args %v", args) + output, cmdErr := commandExecutor(ctx, app, args...).CombinedOutput() + + // check if the command timed out during execution for proper logging + if err := ctx.Err(); err != nil { + return err + } + + // enrich error with output from the command which may contain the actual reason + if cmdErr != nil { + return fmt.Errorf("Command error, stderr: %s, stdout: %s", cmdErr.Error(), string(output)) + } + + log.Info().Msgf("Image copied to target: %s", dest) + return nil +} + +// CreateRepository is empty since repositories are not created for artifact registry +func (g *GenericClient) CreateRepository(ctx context.Context, name string) error { + return nil +} + +func (g *GenericClient) ImageExists(ctx context.Context, imageRef ctypes.ImageReference) bool { + ref := imageRef.DockerReference().String() + if _, found := g.cache.Get(ref); found { + log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in cache") + return true + } + + app := "skopeo" + args := []string{ + "inspect", + "--retry-times", "3", + dockerPrefix + ref, + "--creds", g.Credentials(), + } + + if g.ignoreCert { + args = append(args, "--tls-verify=false") + } + + log.Ctx(ctx).Trace().Str("app", app).Strs("args", args).Msg("executing command to inspect image") + if err := commandExecutor(ctx, app, args...).Run(); err != nil { + log.Trace().Str("ref", ref).Msg("not found in repository") + return false + } + + log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in repository") + + g.cache.Set(ref, "", 1) + + return true +} + +func (g *GenericClient) Endpoint() string { + return g.repository +} + +// IsOrigin returns true if the references origin is from this registry +func (g *GenericClient) IsOrigin(imageRef ctypes.ImageReference) bool { + return strings.HasPrefix(imageRef.DockerReference().String(), g.Endpoint()) +} + +func (g *GenericClient) Credentials() string { + return g.username + ":" + g.password +} + +func (g *GenericClient) DockerConfig() ([]byte, error) { + dockerConfig := DockerConfig{ + AuthConfigs: map[string]AuthConfig{ + g.repository: { + Auth: base64.StdEncoding.EncodeToString([]byte(g.password)), + }, + }, + } + + dockerConfigJson, err := json.Marshal(dockerConfig) + if err != nil { + return []byte{}, err + } + + return dockerConfigJson, nil +} diff --git a/pkg/registry/generic_test.go b/pkg/registry/generic_test.go new file mode 100644 index 00000000..3d6231d4 --- /dev/null +++ b/pkg/registry/generic_test.go @@ -0,0 +1,256 @@ +package registry + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "testing" + "time" + + "github.com/containers/image/v5/transports/alltransports" + "github.com/estahn/k8s-image-swapper/pkg/config" + "github.com/stretchr/testify/assert" +) + +var defaultGenericCfg = config.GENERIC{ + Repository: "localhost", + Username: "user", + Password: "password", + IgnoreCert: true, +} + +func createGenericClient(config config.GENERIC, testName string) (*GenericClient, error) { + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", testName, name, arg) + return testCommandExecutor{ + output: []byte("login successful"), + err: nil, + } + } + return NewGenericClient(config) +} + +func TestNewGenericClientSuccess(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + assert.Equal(t, "localhost", genericClient.repository) + assert.Equal(t, "user", genericClient.username) + assert.Equal(t, "password", genericClient.password) +} + +func TestLoginFailure(t *testing.T) { + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: errors.New("login failure"), + } + } + + _, err := NewGenericClient(defaultGenericCfg) + assert.NotNil(t, err) +} + +func TestImageExistsSuccess(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + imageRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + exists := genericClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, true, exists) +} + +func TestImageExistsInCacheSuccess(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + imageRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + genericClient.cache.Set(imageRef.DockerReference().String(), "123", 123) + time.Sleep(time.Millisecond * 10) + exists := genericClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, true, exists) +} + +func TestImageDoesNotExistsSuccess(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + err: errors.New("Image not found"), + } + } + + imageRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + exists := genericClient.ImageExists(context.Background(), imageRef) + assert.Equal(t, false, exists) +} + +func TestCopyImageSuccess(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + srcRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "user:pass" + destCreds := "user:pass" + + err = genericClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.Nil(t, err) +} + +func TestCopyImageSuccessNoCredentials(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte(""), + err: nil, + } + } + + srcRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "" + destCreds := "" + + err = genericClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.Nil(t, err) +} + +func TestCopyImageFailure(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + curCommandExecutor := commandExecutor + defer func() { commandExecutor = curCommandExecutor }() + + commandExecutor = func(ctx context.Context, name string, arg ...string) ShellCommand { + fmt.Printf("exec.Command() for %v called with %v and %v\n", t.Name(), name, arg) + return testCommandExecutor{ + output: []byte("command not found"), + err: errors.New("copy error"), + } + } + + srcRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + destRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + srcCreds := "user:pass" + destCreds := "user:pass" + + err = genericClient.CopyImage(context.Background(), srcRef, srcCreds, destRef, destCreds) + assert.NotNil(t, err) + assert.Equal(t, "Command error, stderr: copy error, stdout: command not found", err.Error()) +} + +func TestDockerConfigSuccess(t *testing.T) { + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + data, err := genericClient.DockerConfig() + assert.Nil(t, err) + assert.NotNil(t, data) + + dockerConfig := &DockerConfig{} + err = json.Unmarshal(data, dockerConfig) + assert.Nil(t, err) + + for key, authConfig := range dockerConfig.AuthConfigs { + assert.Equal(t, "localhost", key) + assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("password")), authConfig.Auth) + } +} + +func TestIsOrigin(t *testing.T) { + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + imageRef, _ := alltransports.ParseImageName("docker://localhost/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + isOrigin := genericClient.IsOrigin(imageRef) + assert.True(t, isOrigin) +} + +func TestIsNotOrigin(t *testing.T) { + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + imageRef, _ := alltransports.ParseImageName("docker://k8s.gcr.io/ingress-nginx/controller@sha256:9bba603b99bf25f6d117cf1235b6598c16033ad027b143c90fa5b3cc583c5713") + + isOrigin := genericClient.IsOrigin(imageRef) + assert.False(t, isOrigin) +} + +func TestCreateRegistryNilResponse(t *testing.T) { + + genericClient, err := createGenericClient(defaultGenericCfg, t.Name()) + assert.Nil(t, err) + assert.NotNil(t, genericClient) + + err = genericClient.CreateRepository(context.Background(), "repo") + assert.Nil(t, err) +} diff --git a/pkg/registry/inmemory.go b/pkg/registry/inmemory.go deleted file mode 100644 index b2a276fb..00000000 --- a/pkg/registry/inmemory.go +++ /dev/null @@ -1 +0,0 @@ -package registry diff --git a/pkg/secrets/dummy.go b/pkg/secrets/dummy.go index 6ae2e74d..69efc620 100644 --- a/pkg/secrets/dummy.go +++ b/pkg/secrets/dummy.go @@ -17,6 +17,7 @@ func NewDummyImagePullSecretsProvider() ImagePullSecretsProvider { } func (p *DummyImagePullSecretsProvider) SetAuthenticatedRegistries(registries []registry.Client) { + //empty } // GetImagePullSecrets returns an empty ImagePullSecretsResult diff --git a/pkg/types/types.go b/pkg/types/types.go index 647395e5..dd146572 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -8,10 +8,11 @@ const ( RegistryUnknown = iota RegistryAWS RegistryGCP + RegistryGeneric ) func (p Registry) String() string { - return [...]string{"unknown", "aws", "gcp"}[p] + return [...]string{"unknown", "aws", "gcp", "generic"}[p] } func ParseRegistry(p string) (Registry, error) { @@ -20,6 +21,8 @@ func ParseRegistry(p string) (Registry, error) { return RegistryAWS, nil case Registry(RegistryGCP).String(): return RegistryGCP, nil + case Registry(RegistryGeneric).String(): + return RegistryGeneric, nil } return RegistryUnknown, fmt.Errorf("unknown target registry string: '%s', defaulting to unknown", p) } diff --git a/pkg/types/types_test.go b/pkg/types/types_test.go index e6ff93d7..26336515 100644 --- a/pkg/types/types_test.go +++ b/pkg/types/types_test.go @@ -1,6 +1,10 @@ package types -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestParseImageSwapPolicy(t *testing.T) { type args struct { @@ -93,3 +97,31 @@ func TestParseImageCopyPolicy(t *testing.T) { }) } } + +func TestParseAwsRegistry(t *testing.T) { + + registry, err := ParseRegistry("aws") + assert.Nil(t, err) + assert.Equal(t, "aws", registry.String()) +} + +func TestParseGcpRegistry(t *testing.T) { + + registry, err := ParseRegistry("gcp") + assert.Nil(t, err) + assert.Equal(t, "gcp", registry.String()) +} +func TestParseGenericRegistry(t *testing.T) { + + registry, err := ParseRegistry("generic") + assert.Nil(t, err) + assert.Equal(t, "generic", registry.String()) +} + +func TestParseUnknownRegistry(t *testing.T) { + + registry, err := ParseRegistry("not_known") + assert.NotNil(t, err) + assert.Equal(t, "unknown", registry.String()) + assert.Equal(t, "unknown target registry string: 'not_known', defaulting to unknown", err.Error()) +} diff --git a/pkg/webhook/image_copier.go b/pkg/webhook/image_copier.go index 3f2cbfe3..4904ba53 100644 --- a/pkg/webhook/image_copier.go +++ b/pkg/webhook/image_copier.go @@ -3,8 +3,11 @@ package webhook import ( "context" "errors" + "fmt" "os" + "github.com/estahn/k8s-image-swapper/pkg/registry" + "github.com/containers/image/v5/docker/reference" ctypes "github.com/containers/image/v5/types" "github.com/rs/zerolog/log" @@ -87,7 +90,7 @@ func (ic *ImageCopier) run(taskFunc func() error) error { } func (ic *ImageCopier) taskCheckImage() error { - registryClient := ic.imageSwapper.registryClient + registryClient := ic.imageSwapper.destinationRegistryClient imageAlreadyExists := registryClient.ImageExists(ic.context, ic.targetImageRef) && ic.imagePullPolicy != corev1.PullAlways @@ -103,7 +106,7 @@ func (ic *ImageCopier) taskCheckImage() error { func (ic *ImageCopier) taskCreateRepository() error { createRepoName := reference.TrimNamed(ic.sourceImageRef.DockerReference()).String() - return ic.imageSwapper.registryClient.CreateRepository(ic.context, createRepoName) + return ic.imageSwapper.destinationRegistryClient.CreateRepository(ic.context, createRepoName) } func (ic *ImageCopier) taskCopyImage() error { @@ -133,7 +136,24 @@ func (ic *ImageCopier) taskCopyImage() error { // Copy image // TODO: refactor to use structure instead of passing file name / string // - // or transform registryClient creds into auth compatible form, e.g. + // or transform destinationRegistryClient creds into auth compatible form, e.g. // {"auths":{"aws_account_id.dkr.ecr.region.amazonaws.com":{"username":"AWS","password":"..." }}} - return ic.imageSwapper.registryClient.CopyImage(ctx, ic.sourceImageRef, authFile.Name(), ic.targetImageRef, ic.imageSwapper.registryClient.Credentials()) + + //figure out corresponding source + sourceDomain := reference.Domain(ic.sourceImageRef.DockerReference()) + + var sourceRegistryClient registry.Client = nil + for _, sourceClient := range ic.imageSwapper.sourceRegistryClients { + if sourceClient.Endpoint() == sourceDomain { + sourceRegistryClient = sourceClient + break + } + } + if sourceRegistryClient == nil { + return fmt.Errorf("Failed to find source registry when looking for %s", sourceDomain) + } else { + //using authFile + return ic.imageSwapper.destinationRegistryClient.CopyImage(ctx, ic.sourceImageRef, authFile.Name(), ic.targetImageRef, ic.imageSwapper.destinationRegistryClient.Credentials()) + } + } diff --git a/pkg/webhook/image_copier_test.go b/pkg/webhook/image_copier_test.go index a331f6d6..67ac87ca 100644 --- a/pkg/webhook/image_copier_test.go +++ b/pkg/webhook/image_copier_test.go @@ -15,7 +15,9 @@ import ( ) func TestImageCopier_withDeadline(t *testing.T) { + var registryClients []registry.Client mutator := NewImageSwapperWithOpts( + registryClients, nil, ImageCopyDeadline(8*time.Second), ) @@ -67,11 +69,13 @@ func TestImageCopier_tasksTimeout(t *testing.T) { }, }).Return(mock.Anything) - registryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") + targetRegistryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") + srcRegistryClients := []registry.Client{} // image swapper with an instant timeout for testing purpose mutator := NewImageSwapperWithOpts( - registryClient, + srcRegistryClients, + targetRegistryClient, ImageCopyDeadline(0*time.Second), ) diff --git a/pkg/webhook/image_swapper.go b/pkg/webhook/image_swapper.go index ee2aea96..da3b739d 100644 --- a/pkg/webhook/image_swapper.go +++ b/pkg/webhook/image_swapper.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + "github.com/rs/zerolog" + "github.com/alitto/pond" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/transports/alltransports" @@ -70,8 +72,9 @@ func Copier(pool *pond.WorkerPool) Option { // ImageSwapper is a mutator that will download images and change the image name. type ImageSwapper struct { - registryClient registry.Client - imagePullSecretProvider secrets.ImagePullSecretsProvider + sourceRegistryClients []registry.Client + destinationRegistryClient registry.Client + imagePullSecretProvider secrets.ImagePullSecretsProvider // filters defines a list of expressions to remove objects that should not be processed, // by default all objects will be processed @@ -86,26 +89,28 @@ type ImageSwapper struct { } // NewImageSwapper returns a new ImageSwapper initialized. -func NewImageSwapper(registryClient registry.Client, imagePullSecretProvider secrets.ImagePullSecretsProvider, filters []config.JMESPathFilter, imageSwapPolicy types.ImageSwapPolicy, imageCopyPolicy types.ImageCopyPolicy, imageCopyDeadline time.Duration) kwhmutating.Mutator { +func NewImageSwapper(sourceRegistryClients []registry.Client, registryClient registry.Client, imagePullSecretProvider secrets.ImagePullSecretsProvider, filters []config.JMESPathFilter, imageSwapPolicy types.ImageSwapPolicy, imageCopyPolicy types.ImageCopyPolicy, imageCopyDeadline time.Duration) kwhmutating.Mutator { return &ImageSwapper{ - registryClient: registryClient, - imagePullSecretProvider: imagePullSecretProvider, - filters: filters, - copier: pond.New(100, 1000), - imageSwapPolicy: imageSwapPolicy, - imageCopyPolicy: imageCopyPolicy, - imageCopyDeadline: imageCopyDeadline, + sourceRegistryClients: sourceRegistryClients, + destinationRegistryClient: registryClient, + imagePullSecretProvider: imagePullSecretProvider, + filters: filters, + copier: pond.New(100, 1000), + imageSwapPolicy: imageSwapPolicy, + imageCopyPolicy: imageCopyPolicy, + imageCopyDeadline: imageCopyDeadline, } } // NewImageSwapperWithOpts returns a configured ImageSwapper instance -func NewImageSwapperWithOpts(registryClient registry.Client, opts ...Option) kwhmutating.Mutator { +func NewImageSwapperWithOpts(sourceRegistryClient []registry.Client, destinationRegistryClient registry.Client, opts ...Option) kwhmutating.Mutator { swapper := &ImageSwapper{ - registryClient: registryClient, - imagePullSecretProvider: secrets.NewDummyImagePullSecretsProvider(), - filters: []config.JMESPathFilter{}, - imageSwapPolicy: types.ImageSwapPolicyExists, - imageCopyPolicy: types.ImageCopyPolicyDelayed, + sourceRegistryClients: sourceRegistryClient, + destinationRegistryClient: destinationRegistryClient, + imagePullSecretProvider: secrets.NewDummyImagePullSecretsProvider(), + filters: []config.JMESPathFilter{}, + imageSwapPolicy: types.ImageSwapPolicyExists, + imageCopyPolicy: types.ImageCopyPolicyDelayed, } for _, opt := range opts { @@ -120,8 +125,8 @@ func NewImageSwapperWithOpts(registryClient registry.Client, opts ...Option) kwh return swapper } -func NewImageSwapperWebhookWithOpts(registryClient registry.Client, opts ...Option) (webhook.Webhook, error) { - imageSwapper := NewImageSwapperWithOpts(registryClient, opts...) +func NewImageSwapperWebhookWithOpts(sourceRegistryClient []registry.Client, destinationRegistryClient registry.Client, opts ...Option) (webhook.Webhook, error) { + imageSwapper := NewImageSwapperWithOpts(sourceRegistryClient, destinationRegistryClient, opts...) mt := kwhmutating.MutatorFunc(imageSwapper.Mutate) mcfg := kwhmutating.WebhookConfig{ ID: "k8s-image-swapper", @@ -132,8 +137,8 @@ func NewImageSwapperWebhookWithOpts(registryClient registry.Client, opts ...Opti return kwhmutating.NewWebhook(mcfg) } -func NewImageSwapperWebhook(registryClient registry.Client, imagePullSecretProvider secrets.ImagePullSecretsProvider, filters []config.JMESPathFilter, imageSwapPolicy types.ImageSwapPolicy, imageCopyPolicy types.ImageCopyPolicy, imageCopyDeadline time.Duration) (webhook.Webhook, error) { - imageSwapper := NewImageSwapper(registryClient, imagePullSecretProvider, filters, imageSwapPolicy, imageCopyPolicy, imageCopyDeadline) +func NewImageSwapperWebhook(sourceRegistryClients []registry.Client, registryClient registry.Client, imagePullSecretProvider secrets.ImagePullSecretsProvider, filters []config.JMESPathFilter, imageSwapPolicy types.ImageSwapPolicy, imageCopyPolicy types.ImageCopyPolicy, imageCopyDeadline time.Duration) (webhook.Webhook, error) { + imageSwapper := NewImageSwapper(sourceRegistryClients, registryClient, imagePullSecretProvider, filters, imageSwapPolicy, imageCopyPolicy, imageCopyDeadline) mt := kwhmutating.MutatorFunc(imageSwapper.Mutate) mcfg := kwhmutating.WebhookConfig{ ID: "k8s-image-swapper", @@ -166,6 +171,7 @@ func imageNamesWithDigestOrTag(imageName string) (string, error) { // Mutate replaces the image ref. Satisfies mutating.Mutator interface. func (p *ImageSwapper) Mutate(ctx context.Context, ar *kwhmodel.AdmissionReview, obj metav1.Object) (*kwhmutating.MutatorResult, error) { + pod, ok := obj.(*corev1.Pod) if !ok { return &kwhmutating.MutatorResult{}, nil @@ -184,24 +190,19 @@ func (p *ImageSwapper) Mutate(ctx context.Context, ar *kwhmodel.AdmissionReview, for _, containerSet := range containerSets { containers := *containerSet for i, container := range containers { - normalizedName, err := imageNamesWithDigestOrTag(container.Image) - if err != nil { - log.Ctx(lctx).Warn().Msgf("unable to normalize source name %s: %v", container.Image, err) - continue - } - srcRef, err := alltransports.ParseImageName("docker://" + normalizedName) + srcRef, err := p.determineSrcRef(container, lctx) if err != nil { - log.Ctx(lctx).Warn().Msgf("invalid source name %s: %v", normalizedName, err) continue } // skip if the source originates from the target registry - if p.registryClient.IsOrigin(srcRef) { + if p.destinationRegistryClient != nil && p.destinationRegistryClient.IsOrigin(srcRef) { log.Ctx(lctx).Debug().Str("registry", srcRef.DockerReference().String()).Msg("skip due to source and target being the same registry") continue } + //skip if matches filter filterCtx := NewFilterContext(*ar, pod, container) if filterMatch(filterCtx, p.filters) { log.Ctx(lctx).Debug().Msg("skip due to filter condition") @@ -209,58 +210,87 @@ func (p *ImageSwapper) Mutate(ctx context.Context, ar *kwhmodel.AdmissionReview, } targetRef := p.targetRef(srcRef) - targetImage := targetRef.DockerReference().String() - - imageCopierLogger := logger.With(). - Str("source-image", srcRef.DockerReference().String()). - Str("target-image", targetImage). - Logger() - - imageCopierContext := imageCopierLogger.WithContext(lctx) - // create an object responsible for the image copy - imageCopier := ImageCopier{ - sourcePod: pod, - sourceImageRef: srcRef, - targetImageRef: targetRef, - imagePullPolicy: container.ImagePullPolicy, - imageSwapper: p, - context: imageCopierContext, - } - // imageCopyPolicy - switch p.imageCopyPolicy { - case types.ImageCopyPolicyDelayed: - p.copier.Submit(imageCopier.start) - case types.ImageCopyPolicyImmediate: - p.copier.SubmitAndWait(imageCopier.withDeadline().start) - case types.ImageCopyPolicyForce: - imageCopier.withDeadline().start() - case types.ImageCopyPolicyNone: - // do not copy image - default: - panic("unknown imageCopyPolicy") - } + //perform copy + p.copyImage(logger, lctx, pod, srcRef, targetRef, container) + + //swap container image + p.swapContainerImage(lctx, targetRef, containers, i) - // imageSwapPolicy - switch p.imageSwapPolicy { - case types.ImageSwapPolicyAlways: - log.Ctx(lctx).Debug().Str("image", targetImage).Msg("set new container image") - containers[i].Image = targetImage - case types.ImageSwapPolicyExists: - if p.registryClient.ImageExists(lctx, targetRef) { - log.Ctx(lctx).Debug().Str("image", targetImage).Msg("set new container image") - containers[i].Image = targetImage - } else { - log.Ctx(lctx).Debug().Str("image", targetImage).Msg("container image not found in target registry, not swapping") - } - default: - panic("unknown imageSwapPolicy") - } } } return &kwhmutating.MutatorResult{MutatedObject: pod}, nil } +func (p *ImageSwapper) determineSrcRef(container corev1.Container, lctx context.Context) (ctypes.ImageReference, error) { + + normalizedName, err := imageNamesWithDigestOrTag(container.Image) + if err != nil { + log.Ctx(lctx).Warn().Msgf("unable to normalize source name %s: %v", container.Image, err) + return nil, err + } + + srcRef, err := alltransports.ParseImageName("docker://" + normalizedName) + if err != nil { + log.Ctx(lctx).Warn().Msgf("invalid source name %s: %v", normalizedName, err) + return nil, err + } + return srcRef, nil +} + +func (p *ImageSwapper) copyImage(logger zerolog.Logger, lctx context.Context, pod *corev1.Pod, + srcRef ctypes.ImageReference, targetRef ctypes.ImageReference, container corev1.Container) { + + targetImage := targetRef.DockerReference().String() + imageCopierLogger := logger.With(). + Str("source-image", srcRef.DockerReference().String()). + Str("target-image", targetImage). + Logger() + + imageCopierContext := imageCopierLogger.WithContext(lctx) + // create an object responsible for the image copy + imageCopier := ImageCopier{ + sourcePod: pod, + sourceImageRef: srcRef, + targetImageRef: targetRef, + imagePullPolicy: container.ImagePullPolicy, + imageSwapper: p, + context: imageCopierContext, + } + + // imageCopyPolicy + switch p.imageCopyPolicy { + case types.ImageCopyPolicyDelayed: + p.copier.Submit(imageCopier.start) + case types.ImageCopyPolicyImmediate: + p.copier.SubmitAndWait(imageCopier.withDeadline().start) + case types.ImageCopyPolicyForce: + imageCopier.withDeadline().start() + case types.ImageCopyPolicyNone: + // do not copy image + default: + panic("unknown imageCopyPolicy") + } +} +func (p *ImageSwapper) swapContainerImage(lctx context.Context, targetRef ctypes.ImageReference, containers []corev1.Container, i int) { + + targetImage := targetRef.DockerReference().String() + + switch p.imageSwapPolicy { + case types.ImageSwapPolicyAlways: + log.Ctx(lctx).Debug().Str("image", targetImage).Msg("set new container image") + containers[i].Image = targetImage + case types.ImageSwapPolicyExists: + if p.destinationRegistryClient.ImageExists(lctx, targetRef) { + log.Ctx(lctx).Debug().Str("image", targetImage).Msg("set new container image") + containers[i].Image = targetImage + } else { + log.Ctx(lctx).Debug().Str("image", targetImage).Msg("container image not found in target registry, not swapping") + } + default: + panic("unknown imageSwapPolicy") + } +} // filterMatch returns true if one of the filters matches the context func filterMatch(ctx FilterContext, filters []config.JMESPathFilter) bool { @@ -303,8 +333,9 @@ func filterMatch(ctx FilterContext, filters []config.JMESPathFilter) bool { } // targetName returns the reference in the target repository -func (p *ImageSwapper) targetRef(srcRef ctypes.ImageReference) ctypes.ImageReference { - targetImage := fmt.Sprintf("%s/%s", p.registryClient.Endpoint(), srcRef.DockerReference().String()) +func (p *ImageSwapper) targetRef(targetRef ctypes.ImageReference) ctypes.ImageReference { + //targetImage := fmt.Sprintf("%s/%s", p.destinationRegistryClient.Endpoint(), targetRef.DockerReference().String()) + targetImage := fmt.Sprintf("%s/%s", p.destinationRegistryClient.Endpoint(), targetRef.DockerReference().String()) ref, err := alltransports.ParseImageName("docker://" + targetImage) if err != nil { diff --git a/pkg/webhook/image_swapper_test.go b/pkg/webhook/image_swapper_test.go index 66bfcaef..18f04ab0 100644 --- a/pkg/webhook/image_swapper_test.go +++ b/pkg/webhook/image_swapper_test.go @@ -261,7 +261,8 @@ func TestImageSwapper_Mutate(t *testing.T) { }).Return(mock.Anything) } - registryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") + targetRegistryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") + srcRegistryClients := []registry.Client{} admissionReview, _ := readAdmissionReviewFromFile("admissionreview-simple.json") admissionReviewModel := model.NewAdmissionReviewV1(admissionReview) @@ -269,7 +270,8 @@ func TestImageSwapper_Mutate(t *testing.T) { copier := pond.New(1, 1) // TODO: test types.ImageSwapPolicyExists wh, err := NewImageSwapperWebhookWithOpts( - registryClient, + srcRegistryClients, + targetRegistryClient, Copier(copier), ImageSwapPolicy(types.ImageSwapPolicyAlways), ) @@ -322,7 +324,8 @@ func TestImageSwapper_MutateWithImagePullSecrets(t *testing.T) { }, }).Return(mock.Anything) - registryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") + targetRegistryClient, _ := registry.NewMockECRClient(ecrClient, "ap-southeast-2", "123456789.dkr.ecr.ap-southeast-2.amazonaws.com", "123456789", "arn:aws:iam::123456789:role/fakerole") + srcRegistryClients := []registry.Client{} admissionReview, _ := readAdmissionReviewFromFile("admissionreview-imagepullsecrets.json") admissionReviewModel := model.NewAdmissionReviewV1(admissionReview) @@ -367,7 +370,8 @@ func TestImageSwapper_MutateWithImagePullSecrets(t *testing.T) { copier := pond.New(1, 1) // TODO: test types.ImageSwapPolicyExists wh, err := NewImageSwapperWebhookWithOpts( - registryClient, + srcRegistryClients, + targetRegistryClient, ImagePullSecretsProvider(provider), Copier(copier), ImageSwapPolicy(types.ImageSwapPolicyAlways), @@ -388,7 +392,8 @@ func TestImageSwapper_MutateWithImagePullSecrets(t *testing.T) { } func TestImageSwapper_GAR_Mutate(t *testing.T) { - registryClient, _ := registry.NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + targetRegistryClient, _ := registry.NewMockGARClient(nil, "us-central1-docker.pkg.dev/gcp-project-123/main") + srcRegistryClients := []registry.Client{} admissionReview, _ := readAdmissionReviewFromFile("admissionreview-simple.json") admissionReviewModel := model.NewAdmissionReviewV1(admissionReview) @@ -396,7 +401,8 @@ func TestImageSwapper_GAR_Mutate(t *testing.T) { copier := pond.New(1, 1) // TODO: test types.ImageSwapPolicyExists wh, err := NewImageSwapperWebhookWithOpts( - registryClient, + srcRegistryClients, + targetRegistryClient, Copier(copier), ImageSwapPolicy(types.ImageSwapPolicyAlways), ) From f8385c03fd6e5fa9f05ebf1ee5da3f06106a4f65 Mon Sep 17 00:00:00 2001 From: John Mehan Date: Mon, 23 Oct 2023 10:34:45 -0400 Subject: [PATCH 2/3] feat: addition of generic registry --- .gitignore | 1 + .k8s-image-swapper.yml | 18 +++++++++++++----- pkg/registry/client.go | 1 - pkg/registry/generic.go | 1 - 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index c4403711..1f57a1c6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.dll *.so *.dylib +*.iml # Test binary, built with `go test -c` *.test diff --git a/.k8s-image-swapper.yml b/.k8s-image-swapper.yml index 92192824..e99ef5ae 100644 --- a/.k8s-image-swapper.yml +++ b/.k8s-image-swapper.yml @@ -33,9 +33,11 @@ source: #- jmespath: "ends_with(obj.metadata.namespace,'-dev')" # registries: -# dockerio: -# username: -# password: +# - type: "generic" +# generic: +# repository: "repo1.azurecr.io" +# username: "user" +# password: "pass" target: type: aws @@ -98,5 +100,11 @@ target: } ] } -# dockerio: -# quayio: + +#target: +# type: generic +# generic: +# repository: "repo1.azurecr.io" +# username: "user" +# password: "pass" +# ignoreCert: false \ No newline at end of file diff --git a/pkg/registry/client.go b/pkg/registry/client.go index db9f432d..a73e596c 100644 --- a/pkg/registry/client.go +++ b/pkg/registry/client.go @@ -12,7 +12,6 @@ import ( ctypes "github.com/containers/image/v5/types" ) -const implementMe = "implement me" const dockerPrefix = "docker://" // Client provides methods required to be implemented by the various target registry clients, e.g. ECR, Docker, Quay. diff --git a/pkg/registry/generic.go b/pkg/registry/generic.go index 40c671e1..4ab545a1 100644 --- a/pkg/registry/generic.go +++ b/pkg/registry/generic.go @@ -16,7 +16,6 @@ import ( type GenericAPI interface{} type GenericClient struct { - client GenericAPI repository string username string password string From 0fb4c9559dd09d060904c7ed48e8626f3175338d Mon Sep 17 00:00:00 2001 From: InputObject2 <30133702+InputObject2@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:25:42 -0500 Subject: [PATCH 3/3] Added the ability to auth without creds and fixed forcing the source repositories being defined in config --- pkg/config/config.go | 6 ------ pkg/config/config_test.go | 8 ++------ pkg/registry/generic.go | 38 +++++++++++++++++++++++++++--------- pkg/registry/generic_test.go | 2 +- pkg/webhook/image_copier.go | 14 +++++++++---- 5 files changed, 42 insertions(+), 26 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 20e29bed..e3d14085 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -175,11 +175,5 @@ func validateGenericRegistry(r Registry) error { if r.GENERIC.Repository == "" { return errorWithType(r, `requires a field "repository"`) } - if r.GENERIC.Username == "" { - return errorWithType(r, `requires a field "username"`) - } - if r.GENERIC.Password == "" { - return errorWithType(r, `requires a field "password"`) - } return nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index fd88c676..585f9fa9 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -314,9 +314,7 @@ func TestGenericRegistryNoUsername(t *testing.T) { genericRegistry.GENERIC.Username = "" err = CheckRegistryConfiguration(genericRegistry) - assert.NotNil(t, err) - assert.Equal(t, "registry of type \"generic\" requires a field \"username\"", err.Error()) - + assert.Nil(t, err) } func TestGenericRegistryNoPassword(t *testing.T) { @@ -330,9 +328,7 @@ func TestGenericRegistryNoPassword(t *testing.T) { genericRegistry.GENERIC.Password = "" err = CheckRegistryConfiguration(genericRegistry) - assert.NotNil(t, err) - assert.Equal(t, "registry of type \"generic\" requires a field \"password\"", err.Error()) - + assert.Nil(t, err) } func TestGCPRegistryNoLocation(t *testing.T) { diff --git a/pkg/registry/generic.go b/pkg/registry/generic.go index 4ab545a1..348ab93a 100644 --- a/pkg/registry/generic.go +++ b/pkg/registry/generic.go @@ -42,9 +42,12 @@ func NewGenericClient(clientConfig config.GENERIC) (*GenericClient, error) { cache: cache, } - err = genericClient.login() - if err != nil { - return nil, err + // Only call login if username and password are provided + if genericClient.username != "" || genericClient.password != "" { + err = genericClient.login() + if err != nil { + return nil, err + } } return genericClient, nil @@ -157,7 +160,13 @@ func (g *GenericClient) ImageExists(ctx context.Context, imageRef ctypes.ImageRe "inspect", "--retry-times", "3", dockerPrefix + ref, - "--creds", g.Credentials(), + } + + creds := g.Credentials() + if creds == "" { + args = append(args, "--no-creds") + } else { + args = append(args, "--creds", creds) } if g.ignoreCert { @@ -171,7 +180,6 @@ func (g *GenericClient) ImageExists(ctx context.Context, imageRef ctypes.ImageRe } log.Ctx(ctx).Trace().Str("ref", ref).Msg("found in repository") - g.cache.Set(ref, "", 1) return true @@ -187,21 +195,33 @@ func (g *GenericClient) IsOrigin(imageRef ctypes.ImageReference) bool { } func (g *GenericClient) Credentials() string { + if g.username == "" && g.password == "" { + return "" + } return g.username + ":" + g.password } func (g *GenericClient) DockerConfig() ([]byte, error) { + var authConfig AuthConfig + + // Use the Credentials method to determine if credentials are present + creds := g.Credentials() + if creds != "" { + authConfig = AuthConfig{ + Auth: base64.StdEncoding.EncodeToString([]byte(creds)), + } + } + + // either we generate an empty config (no auth passed) or we use the provided one (username and password given) dockerConfig := DockerConfig{ AuthConfigs: map[string]AuthConfig{ - g.repository: { - Auth: base64.StdEncoding.EncodeToString([]byte(g.password)), - }, + g.repository: authConfig, }, } dockerConfigJson, err := json.Marshal(dockerConfig) if err != nil { - return []byte{}, err + return nil, err } return dockerConfigJson, nil diff --git a/pkg/registry/generic_test.go b/pkg/registry/generic_test.go index 3d6231d4..a12830bc 100644 --- a/pkg/registry/generic_test.go +++ b/pkg/registry/generic_test.go @@ -219,7 +219,7 @@ func TestDockerConfigSuccess(t *testing.T) { for key, authConfig := range dockerConfig.AuthConfigs { assert.Equal(t, "localhost", key) - assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("password")), authConfig.Auth) + assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("user:password")), authConfig.Auth) } } diff --git a/pkg/webhook/image_copier.go b/pkg/webhook/image_copier.go index 4904ba53..fac61bce 100644 --- a/pkg/webhook/image_copier.go +++ b/pkg/webhook/image_copier.go @@ -3,7 +3,6 @@ package webhook import ( "context" "errors" - "fmt" "os" "github.com/estahn/k8s-image-swapper/pkg/registry" @@ -150,10 +149,17 @@ func (ic *ImageCopier) taskCopyImage() error { } } if sourceRegistryClient == nil { - return fmt.Errorf("Failed to find source registry when looking for %s", sourceDomain) + // we are not going to copy using creds specified in the config. + log.Ctx(ctx).Trace().Msgf("could not find source registry in config when looking for %s, using default (pod) credentials", sourceDomain) } else { - //using authFile - return ic.imageSwapper.destinationRegistryClient.CopyImage(ctx, ic.sourceImageRef, authFile.Name(), ic.targetImageRef, ic.imageSwapper.destinationRegistryClient.Credentials()) + log.Ctx(ctx).Trace().Msgf("using source registry client from config for domain: %s", sourceDomain) } + // Proceed with the copy, the credentials will either be the source from the config or the image's creds. + err = ic.imageSwapper.destinationRegistryClient.CopyImage(ctx, ic.sourceImageRef, authFile.Name(), ic.targetImageRef, ic.imageSwapper.destinationRegistryClient.Credentials()) + if err != nil { + log.Ctx(ctx).Err(err).Msg("error during image copy") + } + return err + }