diff --git a/evaluate/report/markdown.go b/evaluate/report/markdown.go index 17a196a3..e69250b4 100644 --- a/evaluate/report/markdown.go +++ b/evaluate/report/markdown.go @@ -24,6 +24,9 @@ type Markdown struct { CSVPath string // LogPath holds the path of detailed logs. LogPath string + // SVGPath holds the path of the charted results. + // REMARK The charts will be generated during the export. + SVGPath string // AssessmentPerModel holds AssessmentPerModel map[string]metrics.Assessments @@ -43,6 +46,9 @@ type markdownTemplateContext struct { // markdownTemplate holds the template for a Markdown report. var markdownTemplate = template.Must(template.New("template-report").Parse(bytesutil.StringTrimIndentations(` # Evaluation from {{.DateTime.Format "2006-01-02 15:04:05"}} + + {{"{{<"}} img-blog name="{{$svgPath}}" alt="Bar chart that categorizes all evaluated LLMs."{{">}}"}} + This report was generated by [DevQualityEval benchmark](https://github.com/symflower/eval-dev-quality) in ` + "`" + `version {{.Version}}` + "`" + `. ## Results @@ -81,7 +87,7 @@ func (m Markdown) Format(writer io.Writer) error { category := assessment.Category(m.TotalScore) templateContext.ModelsPerCategory[category] = append(templateContext.ModelsPerCategory[category], model) } - // TODO Generate svg using maybe https://github.com/wcharczuk/go-chart. + // TODO Generate svg using maybe https://github.com/wcharczuk/go-chart and save to SVGPath. return pkgerrors.WithStack(markdownTemplate.Execute(writer, templateContext)) } diff --git a/evaluate/report/markdown_test.go b/evaluate/report/markdown_test.go index 91622495..0af39770 100644 --- a/evaluate/report/markdown_test.go +++ b/evaluate/report/markdown_test.go @@ -44,11 +44,14 @@ func TestMarkdownFormat(t *testing.T) { CSVPath: "some/csv/path.csv", LogPath: "some/log/path.log", + SVGPath: "some/svg/path.svg", }, ExpectedReport: ` # Evaluation from 2000-01-01 00:00:00 + {{< img-blog name="some/svg/path.svg" alt="Bar chart that categorizes all evaluated LLMs.">}} + This report was generated by [DevQualityEval benchmark](https://github.com/symflower/eval-dev-quality) in ` + "`" + `version 1234` + "`" + `. ## Results @@ -81,6 +84,7 @@ func TestMarkdownFormat(t *testing.T) { CSVPath: "some/csv/path.csv", LogPath: "some/log/path.log", + SVGPath: "some/svg/path.svg", TotalScore: 1, AssessmentPerModel: map[string]metrics.Assessments{ @@ -95,6 +99,8 @@ func TestMarkdownFormat(t *testing.T) { ExpectedReport: ` # Evaluation from 2000-01-01 00:00:00 + {{< img-blog name="some/svg/path.svg" alt="Bar chart that categorizes all evaluated LLMs.">}} + This report was generated by [DevQualityEval benchmark](https://github.com/symflower/eval-dev-quality) in ` + "`" + `version 1234` + "`" + `. ## Results