diff --git a/cmd/eval-dev-quality/cmd/evaluate.go b/cmd/eval-dev-quality/cmd/evaluate.go index acf8638ac..9faed007b 100644 --- a/cmd/eval-dev-quality/cmd/evaluate.go +++ b/cmd/eval-dev-quality/cmd/evaluate.go @@ -284,6 +284,7 @@ func (command *Evaluate) Execute(args []string) (err error) { CSVPath: csvReportPath, LogPath: logFilePath, + SVGPath: filepath.Join(command.ResultPath, "categories.svg"), AssessmentPerModel: assessmentsPerModel, TotalScore: totalScore, diff --git a/cmd/eval-dev-quality/cmd/evaluate_test.go b/cmd/eval-dev-quality/cmd/evaluate_test.go index 1a782a7d9..d24ec524c 100644 --- a/cmd/eval-dev-quality/cmd/evaluate_test.go +++ b/cmd/eval-dev-quality/cmd/evaluate_test.go @@ -84,6 +84,10 @@ func TestEvaluateExecute(t *testing.T) { } }, ExpectedResultFiles: map[string]func(t *testing.T, repositoryPath string, filePath string, data string){ + "categories.svg": func(t *testing.T, resultPath string, filePath, data string) { + assert.Contains(t, data, "No Excess Response") // Assert "no excess" category is present. + assert.Contains(t, data, "1") // Assert the Y-axis label is at least one for one model in that category. + }, "evaluation.csv": func(t *testing.T, repositoryPath, filePath, data string) { assert.Equal(t, bytesutil.StringTrimIndentations(` model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-not-empty,response-with-code @@ -93,6 +97,7 @@ func TestEvaluateExecute(t *testing.T) { "evaluation.log": nil, "report.md": func(t *testing.T, resultPath string, filePath, data string) { // Ensure the report links to the CSV file and logs. + assert.Contains(t, data, filepath.Join(resultPath, "categories.svg")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.csv")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.log")) }, @@ -113,6 +118,10 @@ func TestEvaluateExecute(t *testing.T) { } }, ExpectedResultFiles: map[string]func(t *testing.T, repositoryPath string, filePath string, data string){ + "categories.svg": func(t *testing.T, resultPath string, filePath, data string) { + assert.Contains(t, data, "No Excess Response") // Assert "no excess" category is present. + assert.Contains(t, data, "1") // Assert the Y-axis label is at least one for one model in that category. + }, "evaluation.csv": func(t *testing.T, repositoryPath, filePath, data string) { assert.Equal(t, bytesutil.StringTrimIndentations(` model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-not-empty,response-with-code @@ -123,6 +132,7 @@ func TestEvaluateExecute(t *testing.T) { "evaluation.log": nil, "report.md": func(t *testing.T, resultPath string, filePath, data string) { // Ensure the report links to the CSV file and logs. + assert.Contains(t, data, filepath.Join(resultPath, "categories.svg")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.csv")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.log")) }, @@ -150,6 +160,10 @@ func TestEvaluateExecute(t *testing.T) { } }, ExpectedResultFiles: map[string]func(t *testing.T, repositoryPath string, filePath string, data string){ + "categories.svg": func(t *testing.T, resultPath string, filePath, data string) { + assert.Contains(t, data, "No Excess Response") // Assert "no excess" category is present. + assert.Contains(t, data, "1") // Assert the Y-axis label is at least one for one model in that category. + }, "evaluation.csv": func(t *testing.T, repositoryPath, filePath, data string) { assert.Equal(t, bytesutil.StringTrimIndentations(` model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-not-empty,response-with-code @@ -159,6 +173,7 @@ func TestEvaluateExecute(t *testing.T) { "evaluation.log": nil, "report.md": func(t *testing.T, resultPath string, filePath, data string) { // Ensure the report links to the CSV file and logs. + assert.Contains(t, data, filepath.Join(resultPath, "categories.svg")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.csv")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.log")) }, @@ -180,6 +195,10 @@ func TestEvaluateExecute(t *testing.T) { } }, ExpectedResultFiles: map[string]func(t *testing.T, repositoryPath string, filePath string, data string){ + "categories.svg": func(t *testing.T, resultPath string, filePath, data string) { + assert.Contains(t, data, "No Excess Response") // Assert "no excess" category is present. + assert.Contains(t, data, "1") // Assert the Y-axis label is at least one for one model in that category. + }, "evaluation.csv": func(t *testing.T, repositoryPath, filePath, data string) { assert.Equal(t, bytesutil.StringTrimIndentations(` model,language,repository,score,coverage-statement,files-executed,response-no-error,response-no-excess,response-not-empty,response-with-code @@ -189,6 +208,7 @@ func TestEvaluateExecute(t *testing.T) { "evaluation.log": nil, "report.md": func(t *testing.T, resultPath string, filePath, data string) { // Ensure the report links to the CSV file and logs. + assert.Contains(t, data, filepath.Join(resultPath, "categories.svg")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.csv")) assert.Contains(t, data, filepath.Join(resultPath, "evaluation.log")) }, diff --git a/evaluate/report/markdown.go b/evaluate/report/markdown.go index 775112857..ed9b01b24 100644 --- a/evaluate/report/markdown.go +++ b/evaluate/report/markdown.go @@ -4,10 +4,12 @@ import ( "io" "os" "path/filepath" + "strconv" "text/template" "time" pkgerrors "github.com/pkg/errors" + "github.com/wcharczuk/go-chart/v2" "github.com/zimmski/osutil/bytesutil" "github.com/symflower/eval-dev-quality/evaluate/metrics" @@ -24,6 +26,9 @@ type Markdown struct { CSVPath string // LogPath holds the path of detailed logs. LogPath string + // SVGPath holds the path of the charted results. + // REMARK The charts will be generated automatically during the export if this path is set. + SVGPath string // AssessmentPerModel holds AssessmentPerModel map[string]metrics.Assessments @@ -44,6 +49,11 @@ type markdownTemplateContext struct { var markdownTemplate = template.Must(template.New("template-report").Parse(bytesutil.StringTrimIndentations(` # Evaluation from {{.DateTime.Format "2006-01-02 15:04:05"}} + {{ with $svgPath := .SVGPath -}} + ![Bar chart that categorizes all evaluated models.]({{$svgPath}}) + + {{ end -}} + This report was generated by [DevQualityEval benchmark](https://github.com/symflower/eval-dev-quality) in ` + "`" + `version {{.Version}}` + "`" + `. ## Results @@ -69,6 +79,52 @@ var markdownTemplate = template.Must(template.New("template-report").Parse(bytes {{- end -}} `))) +// barChartModelsPerCategoriesSVG generates a bar chart showing models per category and writes it out as an SVG. +func barChartModelsPerCategoriesSVG(writer io.Writer, categories []*metrics.AssessmentCategory, modelsPerCategory map[*metrics.AssessmentCategory][]string) error { + bars := make([]chart.Value, 0, len(categories)) + maxCount := 0 + for _, category := range categories { + count := len(modelsPerCategory[category]) + if count > maxCount { + maxCount = count + } + if count == 0 { + continue + } + + bars = append(bars, chart.Value{ + Label: category.Name, + Value: float64(count), + }) + } + ticks := make([]chart.Tick, maxCount+1) + for i := range ticks { + ticks[i] = chart.Tick{ + Value: float64(i), + Label: strconv.Itoa(i), + } + } + graph := chart.BarChart{ + Title: "Models per Category", + Bars: bars, + YAxis: chart.YAxis{ + Ticks: ticks, + }, + + Background: chart.Style{ + Padding: chart.Box{ + Top: 60, + Bottom: 40, + }, + }, + Height: 300, + Width: (len(bars) + 2) * 60, + BarWidth: 60, + } + + return pkgerrors.WithStack(graph.Render(chart.SVG, writer)) +} + // Format formats the markdown values in the template to the given writer. func (m Markdown) Format(writer io.Writer) error { templateContext := markdownTemplateContext{ @@ -80,7 +136,24 @@ func (m Markdown) Format(writer io.Writer) error { category := assessment.Category(m.TotalScore) templateContext.ModelsPerCategory[category] = append(templateContext.ModelsPerCategory[category], model) } - // TODO Generate svg using maybe https://github.com/wcharczuk/go-chart. + + if m.SVGPath == "" { + return pkgerrors.WithStack(markdownTemplate.Execute(writer, templateContext)) + + } + + svgFile, err := os.Create(m.SVGPath) + if err != nil { + return pkgerrors.WithStack(err) + } + defer func() { + if err := svgFile.Close(); err != nil { + panic(err) + } + }() + if err := barChartModelsPerCategoriesSVG(svgFile, metrics.AllAssessmentCategories, templateContext.ModelsPerCategory); err != nil { + return pkgerrors.WithStack(err) + } return pkgerrors.WithStack(markdownTemplate.Execute(writer, templateContext)) } @@ -95,6 +168,12 @@ func (t Markdown) WriteToFile(path string) (err error) { if err != nil { return err } + if t.SVGPath != "" { + t.SVGPath, err = filepath.Abs(t.SVGPath) + if err != nil { + return err + } + } if err = os.MkdirAll(filepath.Base(path), 0755); err != nil { return pkgerrors.WithStack(err) diff --git a/evaluate/report/markdown_test.go b/evaluate/report/markdown_test.go index 27093aaf6..c61e80ec6 100644 --- a/evaluate/report/markdown_test.go +++ b/evaluate/report/markdown_test.go @@ -2,33 +2,75 @@ package report import ( "bytes" + "os" + "path/filepath" + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/symflower/eval-dev-quality/evaluate/metrics" "github.com/zimmski/osutil/bytesutil" + + "github.com/symflower/eval-dev-quality/evaluate/metrics" ) +// validateFileContent asserts that the file content matches the content of the given file path. +// The expected file is created if it does not exist. If the contents don't match, the actual content is written to disk alongside the expected file. +func validateFileContent(t *testing.T, expectedFilePath string, actualFileContent string) { + require.NotEmpty(t, expectedFilePath, "expected file path cannot be empty") + expectedContent, err := os.ReadFile(expectedFilePath) + if err != nil { + // Create the file if it does not exist already to make it easy to add new cases. + require.ErrorIs(t, err, os.ErrNotExist) + expectedContent = []byte("TODO") + require.NoError(t, os.WriteFile(expectedFilePath, expectedContent, 0644)) + t.Logf("expected file %q does not exist yet, created it for you", expectedFilePath) + } + + if !assert.Equalf(t, string(expectedContent), actualFileContent, "actual content:\n%s", actualFileContent) { + extension := filepath.Ext(expectedFilePath) + actualFile := strings.TrimSuffix(expectedFilePath, extension) + "_actual" + extension + require.NoError(t, os.WriteFile(actualFile, []byte(actualFileContent), 0644)) + t.Logf("written actual file content for failing %q to %q", t.Name(), actualFile) + } +} + func TestMarkdownFormat(t *testing.T) { type testCase struct { Name string + // Markdown holds the Markdown values. + // REMARK Do not set the SVG path in the tests as it is set to a temporary file automatically. Markdown Markdown ExpectedReport string - ExpectedError error + // ExpectedSVGFile is the path to the reference file for the generated SVG content. + // REMARK If no SVG reference file is set, none will be generated in the template. + ExpectedSVGFile string + ExpectedError error } validate := func(t *testing.T, tc *testCase) { t.Run(tc.Name, func(t *testing.T) { + temporaryDirectory := t.TempDir() + if tc.ExpectedSVGFile != "" { + tc.Markdown.SVGPath = filepath.Join(temporaryDirectory, "test.svg") + } + var buffer bytes.Buffer actualError := tc.Markdown.Format(&buffer) assert.Equal(t, tc.ExpectedError, actualError) actualReport := buffer.String() + actualReport = strings.ReplaceAll(actualReport, temporaryDirectory, "$TEST_DIR") assert.Equalf(t, bytesutil.StringTrimIndentations(tc.ExpectedReport), actualReport, "Full output:\n%s", actualReport) + + if tc.ExpectedSVGFile != "" { + actualSVGContent, err := os.ReadFile(tc.Markdown.SVGPath) + assert.NoError(t, err) + validateFileContent(t, tc.ExpectedSVGFile, string(actualSVGContent)) + } }) } @@ -94,6 +136,8 @@ func TestMarkdownFormat(t *testing.T) { ExpectedReport: ` # Evaluation from 2000-01-01 00:00:00 + ![Bar chart that categorizes all evaluated models.]($TEST_DIR/test.svg) + This report was generated by [DevQualityEval benchmark](https://github.com/symflower/eval-dev-quality) in ` + "`" + `version 1234` + "`" + `. ## Results @@ -125,5 +169,65 @@ func TestMarkdownFormat(t *testing.T) { - ` + "`ModelNoCode`" + ` `, + ExpectedSVGFile: "testdata/two_models.svg", + }) +} + +func TestBarChartModelsPerCategoriesSVG(t *testing.T) { + type testCase struct { + Name string + + Categories []*metrics.AssessmentCategory + ModelsPerCategory map[*metrics.AssessmentCategory]uint + + ExpectedFile string + ExpectedError error + } + + validate := func(t *testing.T, tc *testCase) { + t.Run(tc.Name, func(t *testing.T) { + var actualSVGContent bytes.Buffer + dummyModelsPerCategory := make(map[*metrics.AssessmentCategory][]string) + for category, count := range tc.ModelsPerCategory { + dummyModelsPerCategory[category] = make([]string, count) + } + + actualError := barChartModelsPerCategoriesSVG(&actualSVGContent, tc.Categories, dummyModelsPerCategory) + assert.Equal(t, tc.ExpectedError, actualError) + + validateFileContent(t, tc.ExpectedFile, actualSVGContent.String()) + }) + } + + validate(t, &testCase{ + Name: "Two Categories", + + Categories: []*metrics.AssessmentCategory{ + metrics.AssessmentCategoryResponseError, + metrics.AssessmentCategoryResponseNoCode, + }, + ModelsPerCategory: map[*metrics.AssessmentCategory]uint{ + metrics.AssessmentCategoryResponseError: 1, + metrics.AssessmentCategoryResponseNoCode: 3, + }, + + ExpectedFile: "testdata/two_categories.svg", + }) + + validate(t, &testCase{ + Name: "All Categories", + + Categories: metrics.AllAssessmentCategories, + ModelsPerCategory: map[*metrics.AssessmentCategory]uint{ + metrics.AssessmentCategoryResponseError: 1, + metrics.AssessmentCategoryResponseEmpty: 2, + metrics.AssessmentCategoryResponseNoCode: 3, + metrics.AssessmentCategoryCodeInvalid: 4, + metrics.AssessmentCategoryCodeExecuted: 5, + metrics.AssessmentCategoryCodeCoverageStatementReached: 6, + metrics.AssessmentCategoryCodeNoExcess: 7, + }, + + ExpectedFile: "testdata/all_categories.svg", }) } diff --git a/evaluate/report/testdata/all_categories.svg b/evaluate/report/testdata/all_categories.svg new file mode 100644 index 000000000..93da31590 --- /dev/null +++ b/evaluate/report/testdata/all_categories.svg @@ -0,0 +1,61 @@ +\nResponseErrorResponseEmptyNo CodeInvalidCodeExecutableCodeStatementCoverageReachedNo ExcessResponse01234567Models per Category \ No newline at end of file diff --git a/evaluate/report/testdata/two_categories.svg b/evaluate/report/testdata/two_categories.svg new file mode 100644 index 000000000..7d1985f7e --- /dev/null +++ b/evaluate/report/testdata/two_categories.svg @@ -0,0 +1,28 @@ +\nResponse ErrorNo Code0123Models per Category \ No newline at end of file diff --git a/evaluate/report/testdata/two_models.svg b/evaluate/report/testdata/two_models.svg new file mode 100644 index 000000000..804f122ce --- /dev/null +++ b/evaluate/report/testdata/two_models.svg @@ -0,0 +1,24 @@ +\nResponse ErrorNo Code01Models per Category \ No newline at end of file diff --git a/go.mod b/go.mod index d01033ae2..1a08c729d 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,16 @@ require ( github.com/sashabaranov/go-openai v1.20.4 github.com/stretchr/testify v1.9.0 github.com/symflower/lockfile v0.0.0-20240419143922-aa3b60940c84 + github.com/wcharczuk/go-chart/v2 v2.1.1 github.com/zimmski/osutil v1.2.0 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 ) require ( github.com/avast/retry-go v3.0.0+incompatible // indirect + github.com/blend/go-sdk v1.20220411.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -23,6 +26,7 @@ require ( github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/yuin/goldmark v1.7.0 // indirect + golang.org/x/image v0.11.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 81a8c8097..9a06e62a6 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,12 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/blend/go-sdk v1.20220411.3 h1:GFV4/FQX5UzXLPwWV03gP811pj7B8J2sbuq+GJQofXc= +github.com/blend/go-sdk v1.20220411.3/go.mod h1:7lnH8fTi6U4i1fArEXRyOIY2E1X4MALg09qsQqY1+ak= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/jessevdk/go-flags v1.5.1-0.20210607101731-3927b71304df h1:JTDw/M13b6dZmEJI/vfcCLENqcjUHi9UBry+R0pjh5Q= github.com/jessevdk/go-flags v1.5.1-0.20210607101731-3927b71304df/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= @@ -31,20 +35,55 @@ github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1 github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/wcharczuk/go-chart/v2 v2.1.1 h1:2u7na789qiD5WzccZsFz4MJWOJP72G+2kUuJoSNqWnE= +github.com/wcharczuk/go-chart/v2 v2.1.1/go.mod h1:CyCAUt2oqvfhCl6Q5ZvAZwItgpQKZOkCJGb+VGv6l14= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA= github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/zimmski/osutil v1.2.0 h1:M0Xau+QdEIN0urgr7RZUqgs5dxsT1pDJ3/mDqd7uexk= github.com/zimmski/osutil v1.2.0/go.mod h1:TZrA1ZvRIeylQ0ECaANmCVlT0WR/62zJxMdQX9SyLvY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= +golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= +golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=