Define a timeout for the "symflower unit-tests" command, so ensure th…

…e execution does not take too much time

Closes #167

model/symflower/symflower.go

@@ -16,12 +16,27 @@ import (
 	"github.com/symflower/eval-dev-quality/util"
 )
 
+// defaultSymbolicExecutionTimeout defines the default symbolic execution timeout.
+var defaultSymbolicExecutionTimeout = time.Duration(10 * time.Minute)
+
 // Model holds a Symflower model using the locally installed CLI.
-type Model struct{}
+type Model struct {
+	// symbolicExecutionTimeout defines the symbolic execution timeout.
+	symbolicExecutionTimeout time.Duration
+}
 
 // NewModel returns a Symflower model.
 func NewModel() (model *Model) {
-	return &Model{}
+	return &Model{
+		symbolicExecutionTimeout: defaultSymbolicExecutionTimeout,
+	}
+}
+
+// NewModelWithTimeout returns a Symflower model with a given timeout.
+func NewModelWithTimeout(timeout time.Duration) (model *Model) {
+	return &Model{
+		symbolicExecutionTimeout: timeout,
+	}
 }
 
 var _ model.Model = (*Model)(nil)
@@ -35,9 +50,12 @@ var _ model.CapabilityWriteTests = (*Model)(nil)
 
 // generateTestsForFile generates test files for the given implementation file in a repository.
 func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, err error) {
+	ctxWithTimeout, cancel := context.WithTimeout(context.Background(), m.symbolicExecutionTimeout)
+	defer cancel()
+
 	start := time.Now()
 
-	output, err := util.CommandWithResult(context.Background(), ctx.Logger, &util.Command{
+	output, err := util.CommandWithResult(ctxWithTimeout, ctx.Logger, &util.Command{
 		Command: []string{
 			tools.SymflowerPath, "unit-tests",
 			"--code-disable-fetch-dependencies",

model/symflower/symflower_test.go

@@ -1,8 +1,10 @@
 package symflower
 
 import (
+	"context"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -72,13 +74,14 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 				assert.ErrorIs(t, tc.ExpectedError, actualError)
 			} else if actualError != nil || tc.ExpectedErrorText != "" {
 				assert.ErrorContains(t, actualError, tc.ExpectedErrorText)
-			}
-			metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment)
+			} else {
+				metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment)
 
-			actualCoverage, actualProblems, err := tc.Language.Execute(logger, repositoryPath)
-			require.NoError(t, err)
-			require.Empty(t, actualProblems)
-			assert.Equal(t, tc.ExpectedCoverage, actualCoverage)
+				actualCoverage, actualProblems, err := tc.Language.Execute(logger, repositoryPath)
+				require.NoError(t, err)
+				require.Empty(t, actualProblems)
+				assert.Equal(t, tc.ExpectedCoverage, actualCoverage)
+			}
 		})
 	}
 
@@ -114,6 +117,26 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 		},
 		ExpectedCoverage: 1,
 	})
+	t.Run("Timeout", func(t *testing.T) {
+		var expectedErrorText string
+		if osutil.IsWindows() {
+			expectedErrorText = context.DeadlineExceeded.Error()
+		} else {
+			expectedErrorText = "signal: killed"
+		}
+
+		validate(t, &testCase{
+			Name: "Timeout",
+
+			Model:    NewModelWithTimeout(time.Duration(1 * time.Millisecond)),
+			Language: &java.Language{},
+
+			RepositoryPath: filepath.Join("..", "..", "testdata", "java", "light"),
+			FilePath:       filepath.Join("src", "main", "java", "com", "eval", "Knapsack.java"),
+
+			ExpectedErrorText: expectedErrorText,
+		})
+	})
 }
 
 func TestExtractGeneratedFilePaths(t *testing.T) {