From 50b217e496085f0ca44f7a5b36f24a19d4a1089c Mon Sep 17 00:00:00 2001 From: Markus Zimmermann Date: Sat, 30 Mar 2024 07:50:15 +0100 Subject: [PATCH 1/4] Disable all linter checks for testing in Go, because those should be part of a different task --- language/golang.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/language/golang.go b/language/golang.go index 3e39930a..f7792458 100644 --- a/language/golang.go +++ b/language/golang.go @@ -54,8 +54,9 @@ func (language *LanguageGolang) Execute(repositoryPath string) (err error) { _, _, err = util.CommandWithResult(&util.Command{ Command: []string{ "go", "test", - "-v", // Output with the maximum information for easier debugging. - "./...", // Always execute all tests of the repository in case multiple test files have been generated. + "-v", // Output with the maximum information for easier debugging. + "-vet=off", // Disable all linter checks, because those should be part of a different task. + "./...", // Always execute all tests of the repository in case multiple test files have been generated. }, Directory: repositoryPath, From 2feaded5179ad0b0e1e7f17376330ac05bda2a53 Mon Sep 17 00:00:00 2001 From: Markus Zimmermann Date: Sat, 30 Mar 2024 09:14:21 +0100 Subject: [PATCH 2/4] Migrate from regular Go testing tool to "gotestsum" for nicer and more convenient outputs, and we need it for an evaluation task I tried to do the "tooling" hack of Kubernetes https://github.com/kubernetes/kubernetes/blob/master/hack/tools/go.mod but Go does not allow me to import packages that are "main" packages. How is that done?! --- .github/workflows/go.yml | 3 +++ Makefile | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f674c61c..a4aba23c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -27,6 +27,9 @@ jobs: chmod +x $PWD/bin/symflower echo "PATH=$PWD/bin:$PATH" >> $GITHUB_ENV + - name: Install testing tools + run: make install-tools-testing + - name: Build run: make install diff --git a/Makefile b/Makefile index 651b6528..0c2fcc31 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,13 @@ install: # [ Date: Sat, 30 Mar 2024 15:53:26 +0100 Subject: [PATCH 3/4] fix, Correct parameter order for checking error in error chain Wupsi, a review would have been good for that code, but through the power of adding new features and more tests, i found that as well. --- language/golang_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/language/golang_test.go b/language/golang_test.go index 4e3ad9b7..0f99d827 100644 --- a/language/golang_test.go +++ b/language/golang_test.go @@ -75,9 +75,11 @@ func TestLanguageGolangExecute(t *testing.T) { actualError := tc.LanguageGolang.Execute(repositoryPath) if tc.ExpectedError != nil { - assert.ErrorIs(t, tc.ExpectedError, actualError) - } else if actualError != nil || tc.ExpectedErrorText != "" { + assert.ErrorIs(t, actualError, tc.ExpectedError) + } else if actualError != nil && tc.ExpectedErrorText != "" { assert.ErrorContains(t, actualError, tc.ExpectedErrorText) + } else { + assert.NoError(t, actualError) } }) } From bb7ceb2c6d3e9a508ee11ec2d3356921ea9632d0 Mon Sep 17 00:00:00 2001 From: Markus Zimmermann Date: Sat, 30 Mar 2024 15:54:39 +0100 Subject: [PATCH 4/4] Make "no tests" a failure for executing tests of a repository, i.e. if no tests are generated it is a failure --- language/errors.go | 11 +++++++++++ language/golang.go | 23 +++++++++++++++++++++-- language/golang_test.go | 2 ++ 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 language/errors.go diff --git a/language/errors.go b/language/errors.go new file mode 100644 index 00000000..57c500d5 --- /dev/null +++ b/language/errors.go @@ -0,0 +1,11 @@ +package language + +import ( + "errors" +) + +// Common errors over all languages. +var ( + // ErrNoTestFound indicates that no tests could be found. + ErrNoTestFound = errors.New("no tests could be found") +) diff --git a/language/golang.go b/language/golang.go index f7792458..d5c54655 100644 --- a/language/golang.go +++ b/language/golang.go @@ -1,8 +1,11 @@ package language import ( + "errors" "os" "path/filepath" + "regexp" + "strconv" "strings" pkgerrors "github.com/pkg/errors" @@ -49,11 +52,16 @@ func (language *LanguageGolang) Files(repositoryPath string) (filePaths []string return filePaths, nil } +var languageGoNoTestsMatch = regexp.MustCompile(`(?m)^DONE (\d+) tests.*in (.+?)$`) + // Execute invokes the language specific testing on the given repository. func (language *LanguageGolang) Execute(repositoryPath string) (err error) { - _, _, err = util.CommandWithResult(&util.Command{ + stdout, _, err := util.CommandWithResult(&util.Command{ Command: []string{ - "go", "test", + "gotestsum", + "--format", "standard-verbose", // Keep formatting consistent. + "--hide-summary", "skipped", // We are not interested in skipped tests, because they are the same as no tests at all. + "--", // Let the real Go "test" tool options begin. "-v", // Output with the maximum information for easier debugging. "-vet=off", // Disable all linter checks, because those should be part of a different task. "./...", // Always execute all tests of the repository in case multiple test files have been generated. @@ -65,5 +73,16 @@ func (language *LanguageGolang) Execute(repositoryPath string) (err error) { return pkgerrors.WithStack(err) } + ms := languageGoNoTestsMatch.FindStringSubmatch(stdout) + if ms == nil { + return pkgerrors.WithStack(errors.New("could not find Go test summary")) + } + testCount, err := strconv.ParseUint(ms[1], 10, 64) + if err != nil { + return pkgerrors.WithStack(err) + } else if testCount == 0 { + return pkgerrors.WithStack(ErrNoTestFound) + } + return nil } diff --git a/language/golang_test.go b/language/golang_test.go index 0f99d827..c1d3fbf7 100644 --- a/language/golang_test.go +++ b/language/golang_test.go @@ -88,6 +88,8 @@ func TestLanguageGolangExecute(t *testing.T) { Name: "No test files", RepositoryPath: "../testdata/golang/plain/", + + ExpectedError: ErrNoTestFound, }) t.Run("With test file", func(t *testing.T) {