Validate log output via dedicated function instead of expecting a log…

… file
symflower · Jul 12, 2024 · 2037283 · 2037283
1 parent bc95810
commit 2037283
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 43 deletions.
diff --git a/evaluate/task/task-code-repair_test.go b/evaluate/task/task-code-repair_test.go
@@ -75,12 +75,10 @@ func TestTaskCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyResponseNoError:               1,
 					},
 				},
-				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-					filepath.Join(string(IdentifierCodeRepair), "mocked-model", "golang", "golang", "mistakes.log"): func(t *testing.T, filePath, data string) {
-						assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00")
-						assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01")
-						assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02")
-					},
+				ValidateLog: func(t *testing.T, data string) {
+					assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00")
+					assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01")
+					assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02")
 				},
 			})
 		}
@@ -141,15 +139,13 @@ func TestTaskCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyResponseNoError:               2,
 					},
 				},
-				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-					filepath.Join(string(IdentifierCodeRepair), "mocked-model", "golang", "golang", "mistakes.log"): func(t *testing.T, filePath, data string) {
-						assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00")
-						assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01")
-						assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02")
-						assert.Contains(t, data, "TestSymflowerTypeUnknown/#00")
-						assert.Contains(t, data, "TestSymflowerTypeUnknown/#01")
-						assert.Contains(t, data, "TestSymflowerTypeUnknown/#02")
-					},
+				ValidateLog: func(t *testing.T, data string) {
+					assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#00")
+					assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#01")
+					assert.Contains(t, data, "TestSymflowerOpeningBracketMissing/#02")
+					assert.Contains(t, data, "TestSymflowerTypeUnknown/#00")
+					assert.Contains(t, data, "TestSymflowerTypeUnknown/#01")
+					assert.Contains(t, data, "TestSymflowerTypeUnknown/#02")
 				},
 			})
 		}
@@ -198,10 +194,8 @@ func TestTaskCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyResponseNoError:               1,
 					},
 				},
-				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-					filepath.Join(string(IdentifierCodeRepair), "mocked-model", "java", "java", "mistakes.log"): func(t *testing.T, filePath, data string) {
-						assert.Contains(t, data, "BUILD SUCCESS")
-					},
+				ValidateLog: func(t *testing.T, data string) {
+					assert.Contains(t, data, "BUILD SUCCESS")
 				},
 			})
 		}
@@ -266,10 +260,8 @@ func TestTaskCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyResponseNoError:               2,
 					},
 				},
-				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-					filepath.Join(string(IdentifierCodeRepair), "mocked-model", "java", "java", "mistakes.log"): func(t *testing.T, filePath, data string) {
-						assert.Contains(t, data, "BUILD SUCCESS")
-					},
+				ValidateLog: func(t *testing.T, data string) {
+					assert.Contains(t, data, "BUILD SUCCESS")
 				},
 			})
 		}

diff --git a/evaluate/task/task-write-test_test.go b/evaluate/task/task-write-test_test.go
@@ -78,11 +78,9 @@ func TestTaskWriteTestsRun(t *testing.T) {
 				"expected 'package', found does",
 				"exit status 1",
 			},
-			ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-				filepath.Join(string(IdentifierWriteTests), "mocked-model", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
-					assert.Contains(t, data, "Evaluating model \"mocked-model\"")
-					assert.Contains(t, data, "PASS: TestTaskB")
-				},
+			ValidateLog: func(t *testing.T, data string) {
+				assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+				assert.Contains(t, data, "PASS: TestTaskB")
 			},
 		})
 	})
@@ -107,13 +105,11 @@ func TestTaskWriteTestsRun(t *testing.T) {
 
 					ExpectedRepositoryAssessment: expectedAssessments,
 					ExpectedProblemContains:      expectedProblems,
-					ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-						filepath.Join(string(IdentifierWriteTests), "mocked-model", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
-							assert.Contains(t, data, "Evaluating model \"mocked-model\"")
-							if assertTestsPass {
-								assert.Contains(t, data, "PASS: TestPlain")
-							}
-						},
+					ValidateLog: func(t *testing.T, data string) {
+						assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+						if assertTestsPass {
+							assert.Contains(t, data, "PASS: TestPlain")
+						}
 					},
 				})
 			}

diff --git a/evaluate/task/test-integration/task_test.go b/evaluate/task/test-integration/task_test.go
@@ -64,13 +64,11 @@ func TestTaskWriteTestsRun(t *testing.T) {
 				metrics.AssessmentKeyResponseWithCode:                   1,
 			},
 		},
-		ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
-			filepath.Join(string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain.log"): func(t *testing.T, filePath, data string) {
-				assert.Contains(t, data, "Evaluating model \"symflower/symbolic-execution\"")
-				assert.Contains(t, data, "Generated 1 test")
-				assert.Contains(t, data, "PASS: TestSymflowerPlain")
-				assert.Contains(t, data, "Evaluated model \"symflower/symbolic-execution\"")
-			},
+		ValidateLog: func(t *testing.T, data string) {
+			assert.Contains(t, data, "Evaluating model \"symflower/symbolic-execution\"")
+			assert.Contains(t, data, "Generated 1 test")
+			assert.Contains(t, data, "PASS: TestSymflowerPlain")
+			assert.Contains(t, data, "Evaluated model \"symflower/symbolic-execution\"")
 		},
 	})
 }
diff --git a/evaluate/task/testing/task.go b/evaluate/task/testing/task.go
@@ -30,6 +30,7 @@ type TestCaseTask struct {
 	ExpectedResultFiles          map[string]func(t *testing.T, filePath string, data string)
 	ExpectedProblemContains      []string
 	ExpectedError                error
+	ValidateLog                  func(t *testing.T, data string)
 }
 
 type createRepositoryFunction func(logger *log.Logger, testDataPath string, repositoryPathRelative string) (repository evaltask.Repository, cleanup func(), err error)
@@ -78,7 +79,7 @@ func (tc *TestCaseTask) Validate(t *testing.T, createRepository createRepository
 		require.NoError(t, err)
 	}
 	sort.Strings(actualResultFiles)
-	expectedResultFiles := make([]string, 0, len(tc.ExpectedResultFiles))
+	var expectedResultFiles []string
 	for filePath, validate := range tc.ExpectedResultFiles {
 		expectedResultFiles = append(expectedResultFiles, filePath)
 
@@ -91,4 +92,8 @@ func (tc *TestCaseTask) Validate(t *testing.T, createRepository createRepository
 	}
 	sort.Strings(expectedResultFiles)
 	assert.Equal(t, expectedResultFiles, actualResultFiles)
+
+	if tc.ValidateLog != nil {
+		tc.ValidateLog(t, logOutput.String())
+	}
 }