Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ability to select repositories in CLI #9

Closed
wants to merge 1 commit into from
Closed

Ability to select repositories in CLI #9

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 36 additions & 2 deletions cmd/eval-symflower-codegen-testing/cmd/evaluate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"log"
"os"
"path/filepath"
"sort"
"strings"
Expand All @@ -20,6 +21,8 @@ type Evaluate struct {
Languages []string `long:"language" description:"Evaluate with this language. By default all languages are used."`
// Models determines which models should be used for the evaluation, or empty if all models should be used.
Models []string `long:"model" description:"Evaluate with this model. By default all models are used."`
// Repositories determines which repository should be used for the evaluation, or empty if all repositories should be used.
Repositories []string `long:"repository" description:"Evaluate with this repository. By default all repositories are used."`
// TestdataPath determines the testdata path where all repositories reside grouped by languages.
TestdataPath string `long:"testdata" description:"Path to the testdata directory where all repositories reside grouped by languages." default:"testdata/"`
}
Expand All @@ -40,6 +43,11 @@ func (command *Evaluate) Execute(args []string) (err error) {
}
sort.Strings(command.Languages)

commandRepositories := map[string]bool{}
for _, r := range command.Repositories {
commandRepositories[r] = true
}

// Gather models.
if len(command.Models) == 0 {
command.Models = maps.Keys(model.Models)
Expand All @@ -64,17 +72,43 @@ func (command *Evaluate) Execute(args []string) (err error) {
}

// Check that models and languages can be evaluated by executing the "plain" repositories.
log.Printf("Checking that models and languages can used for evaluation")
log.Printf("Checking that models and languages can be used for evaluation")
for _, languageID := range command.Languages {
for _, modelID := range command.Models {
model := model.Models[modelID]
language := language.Languages[languageID]

if err := evaluate.EvaluateRepository(model, language, filepath.Join(command.TestdataPath, language.ID(), "plain")); err != nil {
if err := evaluate.EvaluateRepository(model, language, filepath.Join(command.TestdataPath, languageID, "plain")); err != nil {
log.Fatalf("%+v", err)
}
}
}

// Evaluating models and languages.
log.Printf("Evaluating models and languages")
for _, languageID := range command.Languages {
languagePath := filepath.Join(command.TestdataPath, languageID)

repositories, err := os.ReadDir(languagePath)
if err != nil {
log.Fatalf("ERROR: language path %q cannot be accessed: %s", languagePath, err)
}

for _, repository := range repositories {
if !repository.IsDir() || (len(commandRepositories) > 0 && !commandRepositories[repository.Name()]) {
continue
}

for _, modelID := range command.Models {
model := model.Models[modelID]
language := language.Languages[languageID]

if err := evaluate.EvaluateRepository(model, language, filepath.Join(languagePath, repository.Name())); err != nil {
log.Printf("%+v", err)
}
}
}
}

return nil
}
Loading