From 2037283769ba7e6cd3f288bfd7f2289589ca7a5c Mon Sep 17 00:00:00 2001 From: Andreas Humenberger Date: Tue, 9 Jul 2024 08:56:14 +0200 Subject: [PATCH] Validate log output via dedicated function instead of expecting a log file --- evaluate/task/task-code-repair_test.go | 38 ++++++++------------- evaluate/task/task-write-test_test.go | 20 +++++------ evaluate/task/test-integration/task_test.go | 12 +++---- evaluate/task/testing/task.go | 7 +++- 4 files changed, 34 insertions(+), 43 deletions(-) diff --git a/evaluate/task/task-code-repair_test.go b/evaluate/task/task-code-repair_test.go index 36937c3a..a31c89b2 100644 --- a/evaluate/task/task-code-repair_test.go +++ b/evaluate/task/task-code-repair_test.go @@ -75,12 +75,10 @@ func TestTaskCodeRepairRun(t *testing.T) { metrics.AssessmentKeyResponseNoError: 1, }, }, - ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){ - filepath.Join(string(IdentifierCodeRepair), "mocked-model", "golang", "golang", "mistakes.log"): func(t *testing.T, filePath, data string) { - assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00") - assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01") - assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02") - }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00") + assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01") + assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02") }, }) } @@ -141,15 +139,13 @@ func TestTaskCodeRepairRun(t *testing.T) { metrics.AssessmentKeyResponseNoError: 2, }, }, - ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){ - filepath.Join(string(IdentifierCodeRepair), "mocked-model", "golang", "golang", "mistakes.log"): func(t *testing.T, filePath, data string) { - assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00") - assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01") - assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02") - assert.Contains(t, data, "TestSymflowerTypeUnknown/#00") - assert.Contains(t, data, "TestSymflowerTypeUnknown/#01") - assert.Contains(t, data, "TestSymflowerTypeUnknown/#02") - }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00") + assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01") + assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02") + assert.Contains(t, data, "TestSymflowerTypeUnknown/#00") + assert.Contains(t, data, "TestSymflowerTypeUnknown/#01") + assert.Contains(t, data, "TestSymflowerTypeUnknown/#02") }, }) } @@ -198,10 +194,8 @@ func TestTaskCodeRepairRun(t *testing.T) { metrics.AssessmentKeyResponseNoError: 1, }, }, - ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){ - filepath.Join(string(IdentifierCodeRepair), "mocked-model", "java", "java", "mistakes.log"): func(t *testing.T, filePath, data string) { - assert.Contains(t, data, "BUILD SUCCESS") - }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "BUILD SUCCESS") }, }) } @@ -266,10 +260,8 @@ func TestTaskCodeRepairRun(t *testing.T) { metrics.AssessmentKeyResponseNoError: 2, }, }, - ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){ - filepath.Join(string(IdentifierCodeRepair), "mocked-model", "java", "java", "mistakes.log"): func(t *testing.T, filePath, data string) { - assert.Contains(t, data, "BUILD SUCCESS") - }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "BUILD SUCCESS") }, }) } diff --git a/evaluate/task/task-write-test_test.go b/evaluate/task/task-write-test_test.go index da4795ea..bec3b15e 100644 --- a/evaluate/task/task-write-test_test.go +++ b/evaluate/task/task-write-test_test.go @@ -78,11 +78,9 @@ func TestTaskWriteTestsRun(t *testing.T) { "expected 'package', found does", "exit status 1", }, - ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){ - filepath.Join(string(IdentifierWriteTests), "mocked-model", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) { - assert.Contains(t, data, "Evaluating model \"mocked-model\"") - assert.Contains(t, data, "PASS: TestTaskB") - }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "Evaluating model \"mocked-model\"") + assert.Contains(t, data, "PASS: TestTaskB") }, }) }) @@ -107,13 +105,11 @@ func TestTaskWriteTestsRun(t *testing.T) { ExpectedRepositoryAssessment: expectedAssessments, ExpectedProblemContains: expectedProblems, - ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){ - filepath.Join(string(IdentifierWriteTests), "mocked-model", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) { - assert.Contains(t, data, "Evaluating model \"mocked-model\"") - if assertTestsPass { - assert.Contains(t, data, "PASS: TestPlain") - } - }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "Evaluating model \"mocked-model\"") + if assertTestsPass { + assert.Contains(t, data, "PASS: TestPlain") + } }, }) } diff --git a/evaluate/task/test-integration/task_test.go b/evaluate/task/test-integration/task_test.go index 548c4109..24c8efb3 100644 --- a/evaluate/task/test-integration/task_test.go +++ b/evaluate/task/test-integration/task_test.go @@ -64,13 +64,11 @@ func TestTaskWriteTestsRun(t *testing.T) { metrics.AssessmentKeyResponseWithCode: 1, }, }, - ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){ - filepath.Join(string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) { - assert.Contains(t, data, "Evaluating model \"symflower/symbolic-execution\"") - assert.Contains(t, data, "Generated 1 test") - assert.Contains(t, data, "PASS: TestSymflowerPlain") - assert.Contains(t, data, "Evaluated model \"symflower/symbolic-execution\"") - }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "Evaluating model \"symflower/symbolic-execution\"") + assert.Contains(t, data, "Generated 1 test") + assert.Contains(t, data, "PASS: TestSymflowerPlain") + assert.Contains(t, data, "Evaluated model \"symflower/symbolic-execution\"") }, }) } diff --git a/evaluate/task/testing/task.go b/evaluate/task/testing/task.go index f5c41fbe..ee1a0e38 100644 --- a/evaluate/task/testing/task.go +++ b/evaluate/task/testing/task.go @@ -30,6 +30,7 @@ type TestCaseTask struct { ExpectedResultFiles map[string]func(t *testing.T, filePath string, data string) ExpectedProblemContains []string ExpectedError error + ValidateLog func(t *testing.T, data string) } type createRepositoryFunction func(logger *log.Logger, testDataPath string, repositoryPathRelative string) (repository evaltask.Repository, cleanup func(), err error) @@ -78,7 +79,7 @@ func (tc *TestCaseTask) Validate(t *testing.T, createRepository createRepository require.NoError(t, err) } sort.Strings(actualResultFiles) - expectedResultFiles := make([]string, 0, len(tc.ExpectedResultFiles)) + var expectedResultFiles []string for filePath, validate := range tc.ExpectedResultFiles { expectedResultFiles = append(expectedResultFiles, filePath) @@ -91,4 +92,8 @@ func (tc *TestCaseTask) Validate(t *testing.T, createRepository createRepository } sort.Strings(expectedResultFiles) assert.Equal(t, expectedResultFiles, actualResultFiles) + + if tc.ValidateLog != nil { + tc.ValidateLog(t, logOutput.String()) + } }