From 28a8c999fc43851e2e0f6143fb423da235c85d6d Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Thu, 12 Sep 2024 14:41:11 -0700 Subject: [PATCH] refactor: loader only loads project and root blueprints --- blueprint/pkg/blueprint/blueprint.go | 8 +- blueprint/pkg/loader/loader.go | 88 ++++--- blueprint/pkg/loader/loader_test.go | 351 ++++++++++----------------- 3 files changed, 173 insertions(+), 274 deletions(-) diff --git a/blueprint/pkg/blueprint/blueprint.go b/blueprint/pkg/blueprint/blueprint.go index 5955fa59..8ebbfa3e 100644 --- a/blueprint/pkg/blueprint/blueprint.go +++ b/blueprint/pkg/blueprint/blueprint.go @@ -29,7 +29,7 @@ func (b BlueprintFiles) Unify(ctx *cue.Context) (cue.Value, error) { v = v.Unify(bp.Value) } - if err := cuetools.Validate(v, cue.Concrete(true)); err != nil { + if err := cuetools.Validate(v); err != nil { return cue.Value{}, err } @@ -93,12 +93,6 @@ func NewBlueprintFile(ctx *cue.Context, path string, contents []byte, inj inject return BlueprintFile{}, fmt.Errorf("failed to delete version from blueprint file: %w", err) } - v = inj.InjectEnv(v) - - if err := cuetools.Validate(v, cue.Concrete(true)); err != nil { - return BlueprintFile{}, err - } - return BlueprintFile{ Path: path, Value: v, diff --git a/blueprint/pkg/loader/loader.go b/blueprint/pkg/loader/loader.go index cbc007b8..0012bdf8 100644 --- a/blueprint/pkg/loader/loader.go +++ b/blueprint/pkg/loader/loader.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "log/slog" + "os" "path/filepath" "cuelang.org/go/cue" @@ -15,7 +16,7 @@ import ( "github.com/input-output-hk/catalyst-forge/blueprint/pkg/version" "github.com/input-output-hk/catalyst-forge/blueprint/schema" cuetools "github.com/input-output-hk/catalyst-forge/tools/pkg/cue" - "github.com/input-output-hk/catalyst-forge/tools/pkg/walker" + "github.com/spf13/afero" ) //go:generate go run github.com/matryer/moq@latest --pkg mocks --out ./mocks/loader.go . BlueprintLoader @@ -34,16 +35,40 @@ type BlueprintLoader interface { // DefaultBlueprintLoader is the default implementation of the BlueprintLoader type DefaultBlueprintLoader struct { + fs afero.Fs injector injector.Injector logger *slog.Logger - walker walker.ReverseWalker } func (b *DefaultBlueprintLoader) Load(projectPath, gitRootPath string) (blueprint.RawBlueprint, error) { - files, err := b.findBlueprints(projectPath, gitRootPath) + files := make(map[string][]byte) + + pbPath := filepath.Join(projectPath, BlueprintFileName) + pb, err := afero.ReadFile(b.fs, pbPath) if err != nil { - b.logger.Error("Failed to find blueprint files", "error", err) - return blueprint.RawBlueprint{}, fmt.Errorf("failed to find blueprint files: %w", err) + if os.IsNotExist(err) { + b.logger.Warn("No project blueprint file found", "path", pbPath) + } else { + b.logger.Error("Failed to read blueprint file", "path", pbPath, "error", err) + return blueprint.RawBlueprint{}, fmt.Errorf("failed to read blueprint file: %w", err) + } + } else { + files[pbPath] = pb + } + + if projectPath != gitRootPath { + rootPath := filepath.Join(gitRootPath, BlueprintFileName) + rb, err := afero.ReadFile(b.fs, rootPath) + if err != nil { + if os.IsNotExist(err) { + b.logger.Warn("No root blueprint file found", "path", rootPath) + } else { + b.logger.Error("Failed to read blueprint file", "path", rootPath, "error", err) + return blueprint.RawBlueprint{}, fmt.Errorf("failed to read blueprint file: %w", err) + } + } else { + files[rootPath] = rb + } } ctx := cuecontext.New() @@ -81,6 +106,7 @@ func (b *DefaultBlueprintLoader) Load(projectPath, gitRootPath string) (blueprin finalVersion = bps.Version() userBlueprint = userBlueprint.FillPath(cue.ParsePath("version"), finalVersion.String()) + userBlueprint = b.injector.InjectEnv(userBlueprint) finalBlueprint = schema.Unify(userBlueprint) } else { b.logger.Warn("No blueprint files found, using default values") @@ -105,55 +131,27 @@ func (b *DefaultBlueprintLoader) Load(projectPath, gitRootPath string) (blueprin return blueprint.NewRawBlueprint(finalBlueprint), nil } -// findBlueprints searches for blueprint files starting from the startPath and -// ending at the endPath. It returns a map of blueprint file paths to their -// contents or an error if the search fails. -func (b *DefaultBlueprintLoader) findBlueprints(startPath, endPath string) (map[string][]byte, error) { - bps := make(map[string][]byte) - - err := b.walker.Walk( - startPath, - endPath, - func(path string, fileType walker.FileType, openFile func() (walker.FileSeeker, error)) error { - if fileType == walker.FileTypeFile { - if filepath.Base(path) == BlueprintFileName { - reader, err := openFile() - if err != nil { - return err - } - - defer reader.Close() - - data, err := io.ReadAll(reader) - if err != nil { - return err - } - - bps[path] = data - } - } - - return nil - }, - ) - - if err != nil { - return nil, err +// NewDefaultBlueprintLoader creates a new DefaultBlueprintLoader. +func NewDefaultBlueprintLoader(logger *slog.Logger) DefaultBlueprintLoader { + if logger == nil { + logger = slog.New(slog.NewTextHandler(io.Discard, nil)) } - return bps, nil + return DefaultBlueprintLoader{ + fs: afero.NewOsFs(), + injector: injector.NewDefaultInjector(logger), + logger: logger, + } } -// NewDefaultBlueprintLoader creates a new DefaultBlueprintLoader. -func NewDefaultBlueprintLoader(logger *slog.Logger) DefaultBlueprintLoader { +func NewCustomBlueprintLoader(fs afero.Fs, injector injector.Injector, logger *slog.Logger) DefaultBlueprintLoader { if logger == nil { logger = slog.New(slog.NewTextHandler(io.Discard, nil)) } - walker := walker.NewDefaultFSReverseWalker(logger) return DefaultBlueprintLoader{ - injector: injector.NewDefaultInjector(logger), + fs: fs, + injector: injector, logger: logger, - walker: &walker, } } diff --git a/blueprint/pkg/loader/loader_test.go b/blueprint/pkg/loader/loader_test.go index 5269744c..6826b461 100644 --- a/blueprint/pkg/loader/loader_test.go +++ b/blueprint/pkg/loader/loader_test.go @@ -1,29 +1,21 @@ package loader import ( - "errors" "io" "io/fs" "log/slog" - "path/filepath" "strings" "testing" + "cuelang.org/go/cue" "github.com/input-output-hk/catalyst-forge/blueprint/pkg/injector" imocks "github.com/input-output-hk/catalyst-forge/blueprint/pkg/injector/mocks" "github.com/input-output-hk/catalyst-forge/tools/pkg/testutils" - "github.com/input-output-hk/catalyst-forge/tools/pkg/walker" - wmocks "github.com/input-output-hk/catalyst-forge/tools/pkg/walker/mocks" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type fieldTest struct { - fieldPath string - fieldType string - fieldValue any -} - // MockFileSeeker is a mock implementation of the FileSeeker interface. type MockFileSeeker struct { *strings.Reader @@ -52,30 +44,130 @@ func NewMockFileSeeker(s string) MockFileSeeker { } func TestBlueprintLoaderLoad(t *testing.T) { + // defaultInjector := func() injector.Injector { + // return injector.NewInjector( + // slog.New(slog.NewTextHandler(io.Discard, nil)), + // &imocks.EnvGetterMock{ + // GetFunc: func(name string) (string, bool) { + // return "", false + // }, + // }, + // ) + // } + tests := []struct { name string + fs afero.Fs + injector injector.Injector project string gitRoot string files map[string]string - want []fieldTest + cond func(*testing.T, cue.Value) expectErr bool }{ + // { + // name: "no files", + // fs: afero.NewMemMapFs(), + // injector: defaultInjector(), + // project: "/tmp/dir1/dir2", + // gitRoot: "/tmp/dir1/dir2", + // files: map[string]string{}, + // cond: func(t *testing.T, v cue.Value) { + // assert.NoError(t, v.Err()) + // assert.NotEmpty(t, v.LookupPath(cue.ParsePath("version"))) + // }, + // expectErr: false, + // }, + // { + // name: "single file", + // fs: afero.NewMemMapFs(), + // injector: defaultInjector(), + // project: "/tmp/dir1/dir2", + // gitRoot: "/tmp/dir1/dir2", + // files: map[string]string{ + // "/tmp/dir1/dir2/blueprint.cue": ` + // version: "1.0" + // project: { + // name: "test" + // ci: { + // targets: { + // test: { + // privileged: true + // } + // } + // } + // } + // `, + // "/tmp/dir1/.git": "", + // }, + // cond: func(t *testing.T, v cue.Value) { + // assert.NoError(t, v.Err()) + + // field, err := v.LookupPath(cue.ParsePath("project.ci.targets.test.privileged")).Bool() + // require.NoError(t, err) + // assert.Equal(t, true, field) + // }, + // expectErr: false, + // }, + // { + // name: "multiple files", + // fs: afero.NewMemMapFs(), + // injector: defaultInjector(), + // project: "/tmp/dir1/dir2", + // gitRoot: "/tmp/dir1", + // files: map[string]string{ + // "/tmp/dir1/dir2/blueprint.cue": ` + // version: "1.0" + // project: { + // name: "test" + // ci: { + // targets: { + // test: { + // privileged: true + // } + // } + // } + // } + // `, + // "/tmp/dir1/blueprint.cue": ` + // version: "1.1" + // project: ci: { + // targets: { + // test: { + // retries: 3 + // } + // } + // } + // `, + // }, + // cond: func(t *testing.T, v cue.Value) { + // assert.NoError(t, v.Err()) + + // field1, err := v.LookupPath(cue.ParsePath("project.ci.targets.test.privileged")).Bool() + // require.NoError(t, err) + // assert.Equal(t, true, field1) + + // field2, err := v.LookupPath(cue.ParsePath("project.ci.targets.test.retries")).Int64() + // require.NoError(t, err) + // assert.Equal(t, int64(3), field2) + // }, + // expectErr: false, + // }, { - name: "no files", - project: "/tmp/dir1/dir2", - gitRoot: "/tmp/dir1/dir2", - files: map[string]string{}, - want: []fieldTest{ - { - fieldPath: "version", - fieldType: "string", - fieldValue: "1.0.0", // TODO: This may change + name: "with injection", + fs: afero.NewMemMapFs(), + injector: injector.NewInjector( + slog.New(slog.NewTextHandler(io.Discard, nil)), + &imocks.EnvGetterMock{ + GetFunc: func(name string) (string, bool) { + if name == "RETRIES" { + return "5", true + } + + return "", false + }, }, - }, - expectErr: false, - }, - { - name: "single file", + ), project: "/tmp/dir1/dir2", gitRoot: "/tmp/dir1/dir2", files: map[string]string{ @@ -86,7 +178,7 @@ func TestBlueprintLoaderLoad(t *testing.T) { ci: { targets: { test: { - privileged: true + retries: _ @env(name=RETRIES,type=int) } } } @@ -94,60 +186,12 @@ func TestBlueprintLoaderLoad(t *testing.T) { `, "/tmp/dir1/.git": "", }, - want: []fieldTest{ - { - fieldPath: "project.ci.targets.test.privileged", - fieldType: "bool", - fieldValue: true, - }, - }, - expectErr: false, - }, - { - name: "multiple files", - project: "/tmp/dir1/dir2", - gitRoot: "/tmp/dir1", - files: map[string]string{ - "/tmp/dir1/dir2/blueprint.cue": ` - version: "1.0" - project: { - name: "test" - ci: { - targets: { - test: { - privileged: true - } - } - } - } - `, - "/tmp/dir1/blueprint.cue": ` - version: "1.1" - project: ci: { - targets: { - test: { - retries: 3 - } - } - } - `, - }, - want: []fieldTest{ - { - fieldPath: "version", - fieldType: "string", - fieldValue: "1.1.0", - }, - { - fieldPath: "project.ci.targets.test.privileged", - fieldType: "bool", - fieldValue: true, - }, - { - fieldPath: "project.ci.targets.test.retries", - fieldType: "int", - fieldValue: int64(3), - }, + cond: func(t *testing.T, v cue.Value) { + assert.NoError(t, v.Err()) + + field, err := v.LookupPath(cue.ParsePath("project.ci.targets.test.retries")).Int64() + require.NoError(t, err) + assert.Equal(t, int64(5), field) }, expectErr: false, }, @@ -155,57 +199,12 @@ func TestBlueprintLoaderLoad(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - walker := &wmocks.ReverseWalkerMock{ - WalkFunc: func(startPath string, endPath string, callback walker.WalkerCallback) error { - // True when there is no git root, so we simulate only searching for blueprint files in the root path. - if startPath == endPath && len(tt.files) > 0 { - err := callback(filepath.Join(tt.project, "blueprint.cue"), walker.FileTypeFile, func() (walker.FileSeeker, error) { - return NewMockFileSeeker(tt.files[filepath.Join(tt.project, "blueprint.cue")]), nil - }) - - if err != nil { - return err - } - - return nil - } else if startPath == endPath && len(tt.files) == 0 { - return nil - } - - for path, content := range tt.files { - var err error - if content == "" { - err = callback(path, walker.FileTypeDir, func() (walker.FileSeeker, error) { - return nil, nil - }) - } else { - err = callback(path, walker.FileTypeFile, func() (walker.FileSeeker, error) { - return NewMockFileSeeker(content), nil - }) - } - - if errors.Is(err, io.EOF) { - return nil - } else if err != nil { - return err - } - } - - return nil - }, - } + testutils.SetupFS(t, tt.fs, tt.files) loader := DefaultBlueprintLoader{ - injector: injector.NewInjector( - slog.New(slog.NewTextHandler(io.Discard, nil)), - &imocks.EnvGetterMock{ - GetFunc: func(name string) (string, bool) { - return "", false - }, - }, - ), - logger: slog.New(slog.NewTextHandler(io.Discard, nil)), - walker: walker, + fs: tt.fs, + injector: tt.injector, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } bp, err := loader.Load(tt.project, tt.gitRoot) @@ -213,99 +212,7 @@ func TestBlueprintLoaderLoad(t *testing.T) { return } - for _, test := range tt.want { - value := bp.Get(test.fieldPath) - assert.Nil(t, value.Err(), "failed to lookup field %s: %v", test.fieldPath, value.Err()) - - switch test.fieldType { - case "bool": - b, err := value.Bool() - require.NoError(t, err, "failed to get bool value: %v", err) - assert.Equal(t, b, test.fieldValue.(bool)) - case "int": - i, err := value.Int64() - require.NoError(t, err, "failed to get int value: %v", err) - assert.Equal(t, i, test.fieldValue.(int64)) - case "string": - s, err := value.String() - require.NoError(t, err, "failed to get string value: %v", err) - assert.Equal(t, s, test.fieldValue.(string)) - } - } - }) - } -} - -func TestBlueprintLoader_findBlueprints(t *testing.T) { - tests := []struct { - name string - files map[string]string - walkErr error - want map[string][]byte - expectErr bool - }{ - { - name: "simple", - files: map[string]string{ - "/tmp/test1/test2/blueprint.cue": "test1", - "/tmp/test1/foo.bar": "foobar", - "/tmp/test1/blueprint.cue": "test2", - "/tmp/blueprint.cue": "test3", - }, - want: map[string][]byte{ - "/tmp/test1/test2/blueprint.cue": []byte("test1"), - "/tmp/test1/blueprint.cue": []byte("test2"), - "/tmp/blueprint.cue": []byte("test3"), - }, - expectErr: false, - }, - { - name: "no files", - files: map[string]string{ - "/tmp/test1/foo.bar": "foobar", - }, - want: map[string][]byte{}, - expectErr: false, - }, - { - name: "error", - files: map[string]string{ - "/tmp/test1/foo.bar": "foobar", - }, - walkErr: errors.New("error"), - expectErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - walker := &wmocks.ReverseWalkerMock{ - WalkFunc: func(startPath string, endPath string, callback walker.WalkerCallback) error { - for path, content := range tt.files { - err := callback(path, walker.FileTypeFile, func() (walker.FileSeeker, error) { - return NewMockFileSeeker(content), nil - }) - - if err != nil { - return err - } - } - return tt.walkErr - }, - } - - loader := DefaultBlueprintLoader{ - walker: walker, - } - got, err := loader.findBlueprints("/tmp", "/tmp") - if testutils.AssertError(t, err, tt.expectErr, "") { - return - } - - for k, v := range got { - require.Contains(t, tt.want, k) - assert.Equal(t, tt.want[k], v) - } + tt.cond(t, bp.Value()) }) } }