From 11b1d67a9af3a1b3772ad1a8b7166adef6b8de39 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:12:06 +0200 Subject: [PATCH 01/26] Browse tests, retries, coverage and source in reports --- internal/testdrive/report.go | 778 +++++++++++++++++++++++++-- internal/testdrive/report_test.go | 263 ++++++++- internal/testdrive/testdrive_test.go | 4 + 3 files changed, 985 insertions(+), 60 deletions(-) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index 1887cdde..820ef6f6 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -1,63 +1,170 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + package testdrive import ( "errors" "fmt" - "github.com/DataDog/ddtest/internal/testdrive/intake" "html/template" "net/url" "os" "path/filepath" + "slices" + "sort" "strings" "time" + + "github.com/DataDog/ddtest/internal/testdrive/intake" ) const reportFilename = "report.html" -type reportRuntime struct{ Framework, Tracer string } -type reportFact struct{ Label, Value string } type reportCard struct { - Title, Context string - Tests []intake.Test - Coverages []intake.CoverageFact + Title string + Count int + Context string + Tests []reportTest + Coverages []reportCoverage +} + +type reportTest struct { + Label string + Name string + Suite string + SourceFile string + Status string + Tone string + Duration string + Attempts []reportAttempt + CoverageLevel string + CoveredFiles []string + Source reportSource +} + +type reportAttempt struct { + Number int + Status string + Tone string + Duration string + Kind string + ErrorType string + ErrorMessage string + ErrorStack string +} + +type reportCoverage struct { + Name string + Level string + FileCount int + Files []string + SourceFile string + Source reportSource +} + +type reportSource struct { + Start int + End int + Lines []reportSourceLine + Error string +} + +type reportSourceLine struct { + Number int + Code template.HTML +} + +type reportSuite struct { + Name string + Status string + Tone string + Duration string + TestCount int + CoveredCount int + ShowCoverage bool + CoveredFiles []string + Tests []reportSuiteTest +} + +type reportSuiteTest struct { + Name string + Status string + Tone string + Duration string +} + +type reportFact struct { + Label string + Value string + Tone string +} + +type reportArtifact struct { + Title string + Href string } + type reportModel struct { - Headline, Summary string - Facts []reportFact - Cards []reportCard + Headline string + Summary string + Cards []reportCard + Facts []reportFact + Artifacts []reportArtifact + Suites []reportSuite + Tests []reportTest } +type reportRuntime struct{ Framework, Tracer string } + func writeReport(repositoryRoot, sessionDirectory string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) (string, error) { path := filepath.Join(sessionDirectory, reportFilename) file, err := os.Create(path) if err != nil { return "", fmt.Errorf("create testdrive report: %w", err) } - executeErr := testdriveReport.Execute(file, buildReport(repositoryRoot, findings, commandFailed, runtime...)) - if err := errors.Join(executeErr, file.Close()); err != nil { + + model := buildReport(repositoryRoot, findings, commandFailed, runtime...) + executeErr := testdriveReport.Execute(file, model) + closeErr := file.Close() + if err := errors.Join(executeErr, closeErr); err != nil { return "", fmt.Errorf("write testdrive report: %w", err) } return path, nil } -func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) reportModel { + +func buildReport(repositoryRoot string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) reportModel { info := reportRuntime{Framework: "Test command", Tracer: "Isolated installation"} if len(runtime) > 0 { info = runtime[0] } - coverage := "Not reported" - if findings.CoveredTestCount > 0 { - coverage = fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount) - } status := passedFailed(!commandFailed) if findings.TestEventCount == 0 { status = "No test results received" } - model := reportModel{Headline: "Test events received.", Summary: "No findings.", Facts: []reportFact{ - {"Test events", fmt.Sprint(findings.TestEventCount)}, - {"Tests with coverage", coverage}, - {info.Framework, status}, - {"Datadog library", info.Tracer}, - }} + showTestCoverage := findings.CoverageLevel == "test" + showSuiteCoverage := findings.CoverageLevel == "suite" + model := reportModel{ + Headline: "Test events received.", + Summary: "No findings.", + Facts: []reportFact{ + {Label: "Test events", Value: fmt.Sprintf("%d", findings.TestEventCount), Tone: factTone(findings.TestEventCount > 0)}, + {Label: "Tests with coverage", Value: fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount), Tone: factTone(findings.TestCount > 0 && findings.CoveredTestCount == findings.TestCount)}, + {Label: info.Framework, Value: status, Tone: factTone(!commandFailed && findings.TestEventCount > 0)}, + {Label: "Datadog library", Value: info.Tracer, Tone: "good"}, + }, + Artifacts: []reportArtifact{ + {Title: "JSON traffic", Href: "intake/"}, + {Title: "Test output", Href: testOutputFilename}, + }, + Tests: reportTests(repositoryRoot, findings.Tests, showTestCoverage), + } + model.Suites = reportSuites(findings.Tests, showSuiteCoverage) + if findings.CoveredTestCount == 0 { + model.Facts[1].Value = "Not reported" + } + if findings.TestEventCount == 0 { model.Headline = "No test events received." model.Summary = "Check the instrumentation setup." @@ -69,16 +176,30 @@ func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime .. }) } if len(findings.FailedTests) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any tests failed?", Tests: findings.FailedTests}) + model.Cards = append(model.Cards, reportCard{ + Title: "Any tests failed?", Count: len(findings.FailedTests), + Tests: reportTests(repositoryRoot, findings.FailedTests, showTestCoverage), + }) } if len(findings.FlakyTests) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any flaky tests?", Tests: findings.FlakyTests}) + model.Cards = append(model.Cards, reportCard{ + Title: "Any flaky tests?", Count: len(findings.FlakyTests), + Tests: reportTests(repositoryRoot, findings.FlakyTests, showTestCoverage), + }) } if len(findings.SlowTests) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any tests slower than the others?", Context: "Median test time · " + formatDuration(findings.TestDurationMedian), Tests: findings.SlowTests}) + model.Cards = append(model.Cards, reportCard{ + Title: "Any tests slower than the others?", Count: len(findings.SlowTests), + Context: "Median test time · " + formatDuration(findings.TestDurationMedian), + Tests: reportTests(repositoryRoot, findings.SlowTests, showTestCoverage), + }) } if len(findings.BroadCoverage) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any unusually broad test coverage?", Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), Coverages: findings.BroadCoverage}) + model.Cards = append(model.Cards, reportCard{ + Title: "Any unusually broad test coverage?", Count: len(findings.BroadCoverage), + Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), + Coverages: reportCoverages(repositoryRoot, findings.BroadCoverage), + }) } count := len(findings.ConfigurationErrors) + findings.EmptyCoverageEntryCount for _, size := range []int{ @@ -95,13 +216,102 @@ func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime .. return model } -var testdriveReport = template.Must(template.New("testdrive-report").Parse(` -DDTest report - -

{{.Headline}}

{{.Summary}}

-{{range .Cards}}

{{.Title}}

{{.Context}}

{{end}} -

Run details

{{range .Facts}}
{{.Label}}
{{.Value}}
{{end}}
-

Artifacts

`)) +func reportTests(repositoryRoot string, findings []intake.Test, showCoverage bool) []reportTest { + tests := make([]reportTest, 0, len(findings)) + for _, finding := range findings { + label := finding.Name + if finding.Suite != "" { + label = finding.Suite + " › " + finding.Name + } + status, tone := testDisplayStatus(finding) + test := reportTest{ + Label: label, Name: finding.Name, Suite: finding.Suite, + SourceFile: finding.SourceFile, Status: status, Tone: tone, + Duration: formatDuration(findingDuration(finding)), + Attempts: make([]reportAttempt, 0, len(finding.Attempts)), + Source: readSource(repositoryRoot, finding.SourceFile, finding.SourceStart, finding.SourceEnd), + } + if showCoverage && finding.CoverageLevel == "test" { + test.CoverageLevel = finding.CoverageLevel + test.CoveredFiles = finding.CoveredFiles + } + for attemptIndex, attempt := range finding.Attempts { + kind := "Initial run" + if attempt.Retry { + kind = "Retry" + if attempt.RetryReason != "" { + kind += " · " + strings.ReplaceAll(attempt.RetryReason, "_", " ") + } + } + test.Attempts = append(test.Attempts, reportAttempt{ + Number: attemptIndex + 1, Status: displayStatus(attempt.Status), Tone: attemptTone(attempt.Status), + Duration: formatDuration(attempt.Duration), Kind: kind, + ErrorType: attempt.ErrorType, ErrorMessage: attempt.ErrorMessage, ErrorStack: attempt.ErrorStack, + }) + } + tests = append(tests, test) + } + return tests +} + +func reportCoverages(repositoryRoot string, findings []intake.CoverageFact) []reportCoverage { + coverages := make([]reportCoverage, 0, len(findings)) + for _, finding := range findings { + files := slices.Clone(finding.Files) + slices.Sort(files) + coverage := reportCoverage{ + Name: finding.Name, Level: finding.Level, FileCount: finding.FileCount, + Files: files, SourceFile: finding.SourceFile, + } + if finding.Level == "test" { + coverage.Source = readSource(repositoryRoot, finding.SourceFile, finding.SourceStart, finding.SourceEnd) + } + coverages = append(coverages, coverage) + } + return coverages +} + +func reportSuites(tests []intake.Test, showCoverage bool) []reportSuite { + byName := make(map[string]*reportSuite) + durations := make(map[string]time.Duration) + for _, test := range tests { + name := test.Suite + if name == "" { + name = "Unknown suite" + } + suite, found := byName[name] + if !found { + suite = &reportSuite{Name: name, Status: "Passed", Tone: "good", ShowCoverage: showCoverage} + byName[name] = suite + } + status, tone := testDisplayStatus(test) + if status == "Fail" { + suite.Status = "Failed" + suite.Tone = "attention" + } else if status == "Flaky" && suite.Status != "Failed" { + suite.Status = "Flaky" + suite.Tone = "attention" + } + suite.TestCount++ + durations[name] += findingDuration(test) + if showCoverage && test.CoverageLevel == "suite" { + suite.CoveredCount++ + suite.CoveredFiles = appendUniqueStrings(suite.CoveredFiles, test.CoveredFiles...) + } + suite.Tests = append(suite.Tests, reportSuiteTest{ + Name: test.Name, Status: status, Tone: tone, + Duration: formatDuration(findingDuration(test)), + }) + } + + suites := make([]reportSuite, 0, len(byName)) + for _, suite := range byName { + suite.Duration = formatDuration(durations[suite.Name]) + suites = append(suites, *suite) + } + sort.Slice(suites, func(i, j int) bool { return suites[i].Name < suites[j].Name }) + return suites +} func testDisplayStatus(test intake.Test) (string, string) { status := test.Status @@ -120,13 +330,6 @@ func testDisplayStatus(test intake.Test) (string, string) { return displayStatus(status), attemptTone(status) } -func attemptTone(status string) string { - if status == "pass" { - return "good" - } - return "attention" -} - func findingDuration(test intake.Test) time.Duration { if test.Duration != 0 || len(test.Attempts) == 0 { return test.Duration @@ -134,6 +337,246 @@ func findingDuration(test intake.Test) time.Duration { return test.Attempts[0].Duration } +func appendUniqueStrings(values []string, additions ...string) []string { + for _, addition := range additions { + if slices.Contains(values, addition) { + continue + } + values = append(values, addition) + } + slices.Sort(values) + return values +} + +func readSource(repositoryRoot, sourceFile string, sourceStart, sourceEnd int) reportSource { + if sourceFile == "" { + return reportSource{Error: "Source file not reported."} + } + if sourceStart < 1 { + return reportSource{Error: "Source line not reported."} + } + path := sourceFile + if !filepath.IsAbs(path) { + path = filepath.Join(repositoryRoot, filepath.FromSlash(path)) + } + contents, err := os.ReadFile(path) + if err != nil { + return reportSource{Error: "Source could not be read: " + err.Error()} + } + lines := strings.Split(strings.ReplaceAll(string(contents), "\r\n", "\n"), "\n") + if sourceStart > len(lines) { + return reportSource{Error: fmt.Sprintf("Source line %d is outside %s.", sourceStart, sourceFile)} + } + if sourceEnd < sourceStart { + if filepath.Ext(path) == ".py" || filepath.Ext(path) == ".rb" { + sourceEnd = min(sourceStart+11, len(lines)) + } else { + sourceEnd = inferJavaScriptTestEnd(lines, sourceStart) + } + } + sourceEnd = min(sourceEnd, len(lines)) + source := reportSource{Start: sourceStart, End: sourceEnd} + inBlockComment := false + for lineIndex := sourceStart - 1; lineIndex < sourceEnd; lineIndex++ { + code := highlightJavaScriptLine(lines[lineIndex], &inBlockComment) + if filepath.Ext(path) == ".py" || filepath.Ext(path) == ".rb" { + code = template.HTML(template.HTMLEscapeString(lines[lineIndex])) + } + source.Lines = append(source.Lines, reportSourceLine{ + Number: lineIndex + 1, + Code: code, + }) + } + return source +} + +func inferJavaScriptTestEnd(lines []string, sourceStart int) int { + parentheses := 0 + sawParenthesis := false + inBlockComment := false + var quote byte + escaped := false + for lineIndex := sourceStart - 1; lineIndex < len(lines); lineIndex++ { + line := lines[lineIndex] + for characterIndex := 0; characterIndex < len(line); characterIndex++ { + character := line[characterIndex] + if inBlockComment { + if character == '*' && characterIndex+1 < len(line) && line[characterIndex+1] == '/' { + inBlockComment = false + characterIndex++ + } + continue + } + if quote != 0 { + if escaped { + escaped = false + continue + } + if character == '\\' { + escaped = true + continue + } + if character == quote { + quote = 0 + } + continue + } + if character == '/' && characterIndex+1 < len(line) { + switch line[characterIndex+1] { + case '/': + characterIndex = len(line) + continue + case '*': + inBlockComment = true + characterIndex++ + continue + } + } + if character == '\'' || character == '"' || character == '`' { + quote = character + continue + } + switch character { + case '(': + parentheses++ + sawParenthesis = true + case ')': + parentheses-- + if sawParenthesis && parentheses == 0 && onlyStatementEnd(line[characterIndex+1:]) { + return lineIndex + 1 + } + } + } + } + return min(sourceStart+19, len(lines)) +} + +func onlyStatementEnd(value string) bool { + value = strings.TrimSpace(value) + return value == "" || value == ";" || strings.HasPrefix(value, "//") +} + +func highlightJavaScriptLine(line string, inBlockComment *bool) template.HTML { + var highlighted strings.Builder + for index := 0; index < len(line); { + if *inBlockComment { + end := strings.Index(line[index:], "*/") + if end < 0 { + writeToken(&highlighted, "comment", line[index:]) + break + } + end += index + 2 + writeToken(&highlighted, "comment", line[index:end]) + *inBlockComment = false + index = end + continue + } + if strings.HasPrefix(line[index:], "//") { + writeToken(&highlighted, "comment", line[index:]) + break + } + if strings.HasPrefix(line[index:], "/*") { + end := strings.Index(line[index+2:], "*/") + if end < 0 { + writeToken(&highlighted, "comment", line[index:]) + *inBlockComment = true + break + } + end += index + 4 + writeToken(&highlighted, "comment", line[index:end]) + index = end + continue + } + if line[index] == '\'' || line[index] == '"' || line[index] == '`' { + end := stringEnd(line, index) + writeToken(&highlighted, "string", line[index:end]) + index = end + continue + } + if isIdentifierStart(line[index]) { + end := index + 1 + for end < len(line) && isIdentifierPart(line[end]) { + end++ + } + word := line[index:end] + class := "" + if javascriptKeywords[word] { + class = "keyword" + } else if javascriptLiterals[word] { + class = "literal" + } + writeToken(&highlighted, class, word) + index = end + continue + } + if line[index] >= '0' && line[index] <= '9' { + end := index + 1 + for end < len(line) && ((line[end] >= '0' && line[end] <= '9') || line[end] == '.') { + end++ + } + writeToken(&highlighted, "number", line[index:end]) + index = end + continue + } + writeToken(&highlighted, "", line[index:index+1]) + index++ + } + return template.HTML(highlighted.String()) +} + +func stringEnd(line string, start int) int { + quote := line[start] + escaped := false + for index := start + 1; index < len(line); index++ { + if escaped { + escaped = false + continue + } + if line[index] == '\\' { + escaped = true + continue + } + if line[index] == quote { + return index + 1 + } + } + return len(line) +} + +func writeToken(output *strings.Builder, class, value string) { + escaped := template.HTMLEscapeString(value) + if class == "" { + output.WriteString(escaped) + return + } + output.WriteString(``) + output.WriteString(escaped) + output.WriteString(``) +} + +func isIdentifierStart(character byte) bool { + return character == '_' || character == '$' || character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' +} + +func isIdentifierPart(character byte) bool { + return isIdentifierStart(character) || character >= '0' && character <= '9' +} + +var javascriptKeywords = map[string]bool{ + "async": true, "await": true, "break": true, "case": true, "catch": true, "class": true, + "const": true, "continue": true, "default": true, "delete": true, "do": true, "else": true, + "export": true, "extends": true, "finally": true, "for": true, "from": true, "function": true, + "if": true, "import": true, "in": true, "instanceof": true, "let": true, "new": true, + "of": true, "return": true, "switch": true, "throw": true, "try": true, "typeof": true, + "var": true, "void": true, "while": true, "with": true, "yield": true, +} + +var javascriptLiterals = map[string]bool{ + "false": true, "null": true, "true": true, "undefined": true, +} + func passedFailed(passed bool) string { if passed { return "Passed" @@ -141,6 +584,13 @@ func passedFailed(passed bool) string { return "Failed" } +func factTone(good bool) string { + if good { + return "good" + } + return "attention" +} + func displayStatus(status string) string { if status == "" { return "Unknown" @@ -148,6 +598,13 @@ func displayStatus(status string) string { return strings.ToUpper(status[:1]) + status[1:] } +func attemptTone(status string) string { + if status == "pass" { + return "good" + } + return "attention" +} + func formatDuration(duration time.Duration) string { if duration < time.Millisecond { return duration.Round(time.Microsecond).String() @@ -155,13 +612,6 @@ func formatDuration(duration time.Duration) string { return duration.Round(time.Millisecond).String() } -func plural(count int, singular, plural string) string { - if count == 1 { - return singular - } - return plural -} - func fileURL(path string) (string, error) { absolutePath, err := filepath.Abs(path) if err != nil { @@ -190,3 +640,237 @@ func absoluteFileURL(absolutePath string) string { func terminalLink(target, label string) string { return "\x1b]8;;" + target + "\x1b\\" + label + "\x1b]8;;\x1b\\" } + +func plural(count int, singular, plural string) string { + if count == 1 { + return singular + } + return plural +} + +var testdriveReport = template.Must(template.New("testdrive-report").Funcs(template.FuncMap{"plural": plural}).Parse(` +{{define "attempts"}} +
+ {{range .Attempts}}
+
+ {{.Status}} + Run {{.Number}} · {{.Kind}} + {{.Duration}} +
+ {{if .ErrorMessage}}

{{if .ErrorType}}{{.ErrorType}}: {{end}}{{.ErrorMessage}}

{{end}} + {{if .ErrorStack}}
Stack trace
{{.ErrorStack}}
{{end}} +
{{end}} +
+{{end}} +{{define "source"}} + {{if .Lines}}
+
Source · lines {{.Start}}–{{.End}}
+
{{range .Lines}}{{.Number}}{{.Code}}{{end}}
+
{{else if .Error}}

{{.Error}}

{{end}} +{{end}} +{{define "covered-files"}} + {{if .}}
+ +
+
{{end}} +{{end}} +{{define "test-detail"}} +
+ {{if .SourceFile}}

{{.SourceFile}}

{{end}} + {{template "attempts" .}} + {{if .CoverageLevel}}
Coverage · {{.CoverageLevel}} level · {{len .CoveredFiles}} {{plural (len .CoveredFiles) "file" "files"}}{{template "covered-files" .CoveredFiles}}
{{end}} + {{template "source" .Source}} +
+{{end}} + + + + + + ddtest · Test Optimization report + + + +
+

ddtest · local testdrive

+

{{.Headline}}

+

{{.Summary}}

+ + + +
+ {{if .Cards}}
+ {{range .Cards}}
+

{{.Title}}

{{if .Context}}{{.Context}}{{end}}{{.Count}}
+
+ {{range .Tests}}
+ {{.Label}}{{.Status}} · {{.Duration}} + {{template "test-detail" .}} +
{{end}} + {{range .Coverages}}
+ {{.Name}}{{.FileCount}} {{plural .FileCount "file" "files"}} · {{.Level}} level +
+ {{if .SourceFile}}

{{.SourceFile}}

{{end}} + {{template "covered-files" .Files}} + {{template "source" .Source}} +
+
{{end}} +
+
{{end}} +
{{end}} + +
+

Run details

+
{{range .Facts}}
{{.Label}}
{{.Value}}
{{end}}
+
{{range .Artifacts}}{{.Title}} ↗{{end}}
+
+
+ + + + +
+ + + +`)) diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index b7e71ab0..24eb790d 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -1,10 +1,17 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + package testdrive import ( "bytes" "os" + "path/filepath" "strings" "testing" + "time" "github.com/DataDog/ddtest/internal/testdrive/intake" ) @@ -42,27 +49,257 @@ func TestReportCountsIndividualFindings(t *testing.T) { } } -func TestStaticReportEscapesFindingsAndDescribesMissingCoverage(t *testing.T) { - path, err := writeReport(t.TempDir(), t.TempDir(), intake.Facts{TestEventCount: 1, TestCount: 1, FailedTests: []intake.Test{{Name: ""}}}, true) - if err != nil { +func TestReportKeepsPythonAndRubySourceEscaped(t *testing.T) { + for _, extension := range []string{".py", ".rb"} { + t.Run(extension, func(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "test"+extension), []byte("value = ''\nassert value\n"), 0644); err != nil { + t.Fatal(err) + } + source := readSource(root, "test"+extension, 1, 0) + if source.Error != "" || len(source.Lines) < 2 { + t.Fatalf("source not rendered: %+v", source) + } + if !strings.Contains(string(source.Lines[0].Code), "<script>") || strings.Contains(string(source.Lines[0].Code), " - - -`)) +//go:embed report.html +var reportHTML string + +var testdriveReport = template.Must(template.New("testdrive-report").Funcs(template.FuncMap{"plural": plural}).Parse(reportHTML)) diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html new file mode 100644 index 00000000..a4743d80 --- /dev/null +++ b/internal/testdrive/report.html @@ -0,0 +1,225 @@ + +{{define "attempts"}} +
+ {{range .Attempts}}
+
+ {{.Status}} + Run {{.Number}} · {{.Kind}} + {{.Duration}} +
+ {{if .ErrorMessage}}

{{if .ErrorType}}{{.ErrorType}}: {{end}}{{.ErrorMessage}}

{{end}} + {{if .ErrorStack}}
Stack trace
{{.ErrorStack}}
{{end}} +
{{end}} +
+{{end}} +{{define "source"}} + {{if .Lines}}
+
Source · lines {{.Start}}–{{.End}}
+
{{range .Lines}}{{.Number}}{{.Code}}{{end}}
+
{{else if .Error}}

{{.Error}}

{{end}} +{{end}} +{{define "covered-files"}} + {{if .}}
+ +
+
{{end}} +{{end}} +{{define "test-detail"}} +
+ {{if .SourceFile}}

{{.SourceFile}}

{{end}} + {{template "attempts" .}} + {{if .CoverageLevel}}
Coverage · {{.CoverageLevel}} level · {{len .CoveredFiles}} {{plural (len .CoveredFiles) "file" "files"}}{{template "covered-files" .CoveredFiles}}
{{end}} + {{template "source" .Source}} +
+{{end}} + + + + + + ddtest · Test Optimization report + + + +
+

ddtest · local testdrive

+

{{.Headline}}

+

{{.Summary}}

+ + + +
+ {{if .Cards}}
+ {{range .Cards}}
+

{{.Title}}

{{if .Context}}{{.Context}}{{end}}{{.Count}}
+
+ {{range .Tests}}
+ {{.Label}}{{.Status}} · {{.Duration}} + {{template "test-detail" .}} +
{{end}} + {{range .Coverages}}
+ {{.Name}}{{.FileCount}} {{plural .FileCount "file" "files"}} · {{.Level}} level +
+ {{if .SourceFile}}

{{.SourceFile}}

{{end}} + {{template "covered-files" .Files}} + {{template "source" .Source}} +
+
{{end}} +
+
{{end}} +
{{end}} + +
+

Run details

+
{{range .Facts}}
{{.Label}}
{{.Value}}
{{end}}
+
{{range .Artifacts}}{{.Title}} ↗{{end}}
+
+
+ + + + +
+ + + From 1e5c4d63c8ab1244b4d0d879f37a98b0603430b9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 29 Sep 2026 11:07:09 +0200 Subject: [PATCH 08/26] Redesign test report with finding cards and filtered result tables --- internal/testdrive/report.go | 73 +- internal/testdrive/report.html | 1948 +++++++++++++++++++++++--- internal/testdrive/report_test.go | 46 +- internal/testdrive/testdrive.go | 2 +- internal/testdrive/testdrive_test.go | 6 +- 5 files changed, 1817 insertions(+), 258 deletions(-) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index b699d7f6..451c62be 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -24,6 +24,7 @@ import ( const reportFilename = "report.html" type reportCard struct { + Kind string Title string Count int Context string @@ -37,7 +38,6 @@ type reportTest struct { Suite string SourceFile string Status string - Tone string Duration string Attempts []reportAttempt CoverageLevel string @@ -48,7 +48,6 @@ type reportTest struct { type reportAttempt struct { Number int Status string - Tone string Duration string Kind string ErrorType string @@ -85,7 +84,6 @@ type reportSourceLine struct { type reportSuite struct { Name string Status string - Tone string Duration string TestCount int CoveredCount int @@ -97,7 +95,6 @@ type reportSuite struct { type reportSuiteTest struct { Name string Status string - Tone string Duration string } @@ -204,6 +201,7 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo } if findings.EmptyCoverageEntryCount > 0 { model.Cards = append(model.Cards, reportCard{ + Kind: "error", Title: "Tracer error: empty coverage entries", Count: findings.EmptyCoverageEntryCount, Context: fmt.Sprintf("%d coverage entries had an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.", findings.EmptyCoverageEntryCount), @@ -211,26 +209,26 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo } if len(findings.FailedTests) > 0 { model.Cards = append(model.Cards, reportCard{ - Title: "Any tests failed?", Count: len(findings.FailedTests), + Kind: "failed", Title: "Failed tests", Context: "Inspect the errors and source behind each failure.", Count: len(findings.FailedTests), Tests: builder.reportTests(findings.FailedTests, showTestCoverage), }) } if len(findings.FlakyTests) > 0 { model.Cards = append(model.Cards, reportCard{ - Title: "Any flaky tests?", Count: len(findings.FlakyTests), + Kind: "flaky", Title: "Flaky tests", Context: "These tests both passed and failed across attempts.", Count: len(findings.FlakyTests), Tests: builder.reportTests(findings.FlakyTests, showTestCoverage), }) } if len(findings.SlowTests) > 0 { model.Cards = append(model.Cards, reportCard{ - Title: "Any tests slower than the others?", Count: len(findings.SlowTests), + Kind: "slow", Title: "Slow tests", Count: len(findings.SlowTests), Context: "Median test time · " + formatDuration(findings.TestDurationMedian), Tests: builder.reportTests(findings.SlowTests, showTestCoverage), }) } if len(findings.BroadCoverage) > 0 { model.Cards = append(model.Cards, reportCard{ - Title: "Any unusually broad test coverage?", Count: len(findings.BroadCoverage), + Kind: "coverage", Title: "Broad coverage", Count: len(findings.BroadCoverage), Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), Coverages: builder.reportCoverages(findings.BroadCoverage), }) @@ -253,20 +251,25 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo func (builder *reportBuilder) reportTests(findings []intake.Test, showCoverage bool) []reportTest { tests := make([]reportTest, 0, len(findings)) for _, finding := range findings { - label := finding.Name + name := finding.Name if finding.Parameters != "" { - label += " " + finding.Parameters + name += " " + finding.Parameters } + label := name if finding.Suite != "" { label = finding.Suite + " › " + label } if finding.Module != "" { label = finding.Module + " › " + label } - status, tone := testDisplayStatus(finding) + status := testDisplayStatus(finding) + suite := finding.Suite + if finding.Module != "" { + suite = finding.Module + " › " + suite + } test := reportTest{ - Label: label, Name: finding.Name, Suite: finding.Suite, - SourceFile: finding.SourceFile, Status: status, Tone: tone, + Label: label, Name: name, Suite: suite, + SourceFile: finding.SourceFile, Status: status, Duration: formatDuration(findingDuration(finding)), Attempts: make([]reportAttempt, 0, len(finding.Attempts)), Source: builder.readSource(finding.SourceFile, finding.SourceStart, finding.SourceEnd), @@ -284,7 +287,7 @@ func (builder *reportBuilder) reportTests(findings []intake.Test, showCoverage b } } test.Attempts = append(test.Attempts, reportAttempt{ - Number: attemptIndex + 1, Status: displayStatus(attempt.Status), Tone: attemptTone(attempt.Status), + Number: attemptIndex + 1, Status: displayStatus(attempt.Status), Duration: formatDuration(attempt.Duration), Kind: kind, ErrorType: attempt.ErrorType, ErrorMessage: attempt.ErrorMessage, ErrorStack: attempt.ErrorStack, }) @@ -321,16 +324,15 @@ func (builder *reportBuilder) reportSuites(tests []intake.Test, coverages []inta name = "Unknown suite" } suite, found := byName[key] - status, tone := testDisplayStatus(test) + status := testDisplayStatus(test) if !found { if test.Module != "" { name = test.Module + " › " + name } - suite = &reportSuite{Name: name, Status: suiteStatus(status), Tone: tone, ShowCoverage: showCoverage} + suite = &reportSuite{Name: name, Status: suiteStatus(status), ShowCoverage: showCoverage} byName[key] = suite } else if suiteStatusRank(status) > suiteStatusRank(suite.Status) { suite.Status = suiteStatus(status) - suite.Tone = tone } suite.TestCount++ durations[key] += findingDuration(test) @@ -339,7 +341,7 @@ func (builder *reportBuilder) reportSuites(tests []intake.Test, coverages []inta testName += " " + test.Parameters } suite.Tests = append(suite.Tests, reportSuiteTest{ - Name: testName, Status: status, Tone: tone, + Name: testName, Status: status, Duration: formatDuration(findingDuration(test)), }) } @@ -407,7 +409,7 @@ func suiteStatusRank(status string) int { } } -func testDisplayStatus(test intake.Test) (string, string) { +func testDisplayStatus(test intake.Test) string { status := test.Status sawPass := status == "pass" sawFailure := status == "fail" @@ -416,12 +418,12 @@ func testDisplayStatus(test intake.Test) (string, string) { sawFailure = sawFailure || attempt.Status == "fail" } if sawPass && sawFailure { - return "Flaky", "attention" + return "Flaky" } if status == "" && len(test.Attempts) > 0 { status = test.Attempts[len(test.Attempts)-1].Status } - return displayStatus(status), attemptTone(status) + return displayStatus(status) } func findingDuration(test intake.Test) time.Duration { @@ -626,22 +628,27 @@ func factTone(good bool) string { return "attention" } -func displayStatus(status string) string { - if status == "" { +// reportStatus provides consistent labels for the report's test and suite filters. +func reportStatus(status string) string { + switch status { + case "Pass", "Passed": + return "Passed" + case "Fail", "Failed": + return "Failed" + case "Skip", "Skipped": + return "Skipped" + case "Flaky": + return "Flaky" + default: return "Unknown" } - return strings.ToUpper(status[:1]) + status[1:] } -func attemptTone(status string) string { - switch status { - case "pass": - return "good" - case "fail": - return "attention" - default: - return "" +func displayStatus(status string) string { + if status == "" { + return "Unknown" } + return strings.ToUpper(status[:1]) + status[1:] } func formatDuration(duration time.Duration) string { @@ -690,4 +697,4 @@ func plural(count int, singular, plural string) string { //go:embed report.html var reportHTML string -var testdriveReport = template.Must(template.New("testdrive-report").Funcs(template.FuncMap{"plural": plural}).Parse(reportHTML)) +var testdriveReport = template.Must(template.New("testdrive-report").Funcs(template.FuncMap{"plural": plural, "reportStatus": reportStatus}).Parse(reportHTML)) diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index a4743d80..e0ac946f 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -1,225 +1,1741 @@ + +{{ define "icon-arrow-repeat" }} + +{{ end }} +{{ define "icon-box-arrow-up-right" }} + +{{ end }} +{{ define "icon-check-circle-fill" }} + +{{ end }} +{{ define "icon-chevron-down" }} + +{{ end }} +{{ define "icon-chevron-right" }} + +{{ end }} +{{ define "icon-exclamation-circle" }} + +{{ end }} +{{ define "icon-file-earmark-text" }} + +{{ end }} +{{ define "icon-question-circle" }} + +{{ end }} +{{ define "icon-search" }} + +{{ end }} +{{ define "icon-skip-forward-circle" }} + +{{ end }} +{{ define "icon-stopwatch" }} + +{{ end }} +{{ define "icon-x-circle-fill" }} + +{{ end }} +{{ define "status" }} + {{ if eq (reportStatus .) "Passed" }} + {{ template "icon-check-circle-fill" }} + {{ else if eq (reportStatus .) "Failed" }} + {{ template "icon-x-circle-fill" }} + {{ else if eq (reportStatus .) "Flaky" }} + {{ template "icon-arrow-repeat" }} + {{ else if eq (reportStatus .) "Skipped" }} + {{ template "icon-skip-forward-circle" }} + {{ else }} + {{ template "icon-question-circle" }} + {{ end }}{{ reportStatus . }} +{{ end }} +{{ define "attempts" }}
- {{range .Attempts}}
-
- {{.Status}} - Run {{.Number}} · {{.Kind}} - {{.Duration}} -
- {{if .ErrorMessage}}

{{if .ErrorType}}{{.ErrorType}}: {{end}}{{.ErrorMessage}}

{{end}} - {{if .ErrorStack}}
Stack trace
{{.ErrorStack}}
{{end}} -
{{end}} +

Attempts

+ {{ range .Attempts }} +
+
+ {{ template "status" .Status }}Run {{ .Number }} · {{ .Kind }}{{ .Duration }} +
+ {{ if .ErrorMessage }} + +
{{if .ErrorType}}{{.ErrorType}}: {{end}}{{.ErrorMessage}}
+ {{ end }} + {{ if .ErrorStack }} +
+ Stack trace +
{{ .ErrorStack }}
+
+ {{ end }} +
+ {{ else }} +

No attempts reported.

+ {{ end }} +
+{{ end }} +{{ define "source" }} +
+

+ {{ if .Lines }} + Source · lines {{ .Start }}–{{ .End }} + {{ else }} + Source + {{ end }} +

+ {{ if .Lines }} + +
{{range .Lines}}{{.Number}}{{.Code}}{{end}}
+ {{ else if .Error }} +

{{ .Error }}

+ {{ end }}
-{{end}} -{{define "source"}} - {{if .Lines}}
-
Source · lines {{.Start}}–{{.End}}
-
{{range .Lines}}{{.Number}}{{.Code}}{{end}}
-
{{else if .Error}}

{{.Error}}

{{end}} -{{end}} -{{define "covered-files"}} - {{if .}}
-
    {{range .}}
  • {{.Name}}{{if .Missing}} · missing source{{end}}
  • {{end}}
-
-
{{end}} -{{end}} -{{define "test-detail"}} +{{ end }} +{{ define "covered-files" }} + {{ if . }} +
+
    + {{ range . }} +
  • {{.Name}}{{if .Missing}} · missing source{{end}}
  • + {{ end }} +
+
+ +
+
+ {{ end }} +{{ end }} +{{ define "test-detail" }}
- {{if .SourceFile}}

{{.SourceFile}}

{{end}} - {{template "attempts" .}} - {{if .CoverageLevel}}
Coverage · {{.CoverageLevel}} level · {{len .CoveredFiles}} {{plural (len .CoveredFiles) "file" "files"}}{{template "covered-files" .CoveredFiles}}
{{end}} - {{template "source" .Source}} +
+ {{ template "attempts" . }} +
+ {{ if .SourceFile }} +

{{ .SourceFile }}

+ {{ end }}{{ template "source" .Source }} +
+
+ {{ if .CoverageLevel }} +
+ + Test coverage · {{ len .CoveredFiles }} + {{ plural (len .CoveredFiles) "file" "files" }} + + {{ template "covered-files" .CoveredFiles }} +
+ {{ end }} +
+{{ end }} +{{ define "pager" }} + -{{end}} +{{ end }} - - - - ddtest · Test Optimization report - - - -
-

ddtest · local testdrive

-

{{.Headline}}

-

{{.Summary}}

- - - -
- {{if .Cards}}
- {{range .Cards}}
-

{{.Title}}

{{if .Context}}{{.Context}}{{end}}{{.Count}}
-
- {{range .Tests}}
- {{.Label}}{{.Status}} · {{.Duration}} - {{template "test-detail" .}} -
{{end}} - {{range .Coverages}}
- {{.Name}}{{.FileCount}} {{plural .FileCount "file" "files"}} · {{.Level}} level -
- {{if .SourceFile}}

{{.SourceFile}}

{{end}} - {{template "covered-files" .Files}} - {{template "source" .Source}} -
-
{{end}} -
-
{{end}} -
{{end}} - -
-

Run details

-
{{range .Facts}}
{{.Label}}
{{.Value}}
{{end}}
-
{{range .Artifacts}}{{.Title}} ↗{{end}}
+ + + + ddtest · Test report + + + + +
+ -
- - - - -
- - + {{ if .Cards }} +
+ {{ range $i, $card := .Cards }} +
+ +
+ {{ end }} +
+ {{ range $i, $card := .Cards }} + + {{ end }} + {{ else }} +
+ {{ if eq .Summary "No findings." }} + {{ template "icon-check-circle-fill" }}No findings in this run. + Explore the results below. + {{ else }} + {{ .Summary }} + {{ end }} +
+ {{ end }} + +
+
+ +
+ +
+ + +
+
+ + + + + + + + + + + + + + + {{ range $i, $test := .Tests }} + + + + + + + + + + + + + + + + {{ end }} +
+ Tests +
#StatusTestFile / suiteDurationAttemptsTest coverage + Details +
{{ $i }}{{ template "status" .Status }} + + +
{{ .SourceFile }}
+
{{ .Suite }}
+
{{ .Duration }}{{ len .Attempts }} + {{ if .CoverageLevel }} + {{ len .CoveredFiles }} files + {{ else }} + Not reported + {{ end }} + + {{ template "icon-chevron-down" }} +
+
+ + {{ template "pager" }} +
+ + + diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index da92b6c3..c597e8e9 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -90,7 +90,7 @@ func TestReportShowsEveryFlakyAttemptErrorAndSource(t *testing.T) { report := renderTestReport(t, repositoryRoot, findings) for _, expected := range []string{ - "Any flaky tests?", + "Flaky tests", "Run 1 · Initial run", "Run 2 · Retry · automatic test retry", "20ms", @@ -129,7 +129,7 @@ func TestReportShowsBroadCoverageFilesAndSource(t *testing.T) { report := renderTestReport(t, repositoryRoot, findings) for _, expected := range []string{ - "Any unusually broad test coverage?", + "Broad coverage", "12 files · test level", "Median covered files · 2", "src/one.js", @@ -303,11 +303,13 @@ func TestReportPreservesModuleAndParameters(t *testing.T) { } } -func TestReportSkippedOutcomesHaveNeutralTone(t *testing.T) { +func TestReportSkippedOutcomesStayDistinctFromFailures(t *testing.T) { test := intake.Test{Name: "skipped", Suite: "suite", Status: "skip", Attempts: []intake.TestRun{{Status: "skip"}}} model := buildReport(t.TempDir(), intake.Facts{Tests: []intake.Test{test}}, false) - if model.Tests[0].Tone != "" || model.Tests[0].Attempts[0].Tone != "" || model.Suites[0].Tone != "" || model.Suites[0].Tests[0].Tone != "" { - t.Fatalf("skipped result uses failure tone: test=%+v suite=%+v", model.Tests[0], model.Suites[0]) + for _, status := range []string{model.Tests[0].Status, model.Tests[0].Attempts[0].Status, model.Suites[0].Status, model.Suites[0].Tests[0].Status} { + if reportStatus(status) != "Skipped" { + t.Fatalf("skipped result is displayed as %q", status) + } } } @@ -480,3 +482,37 @@ func TestAbsoluteFileURL(t *testing.T) { } } } + +func TestReportExplorerKeepsIdentitiesEscapedAndDetailsDistinct(t *testing.T) { + tests := []intake.Test{ + {Name: `same `, Module: "one", Suite: "shared", Parameters: `{"case":1}`, Status: "pass"}, + {Name: `same `, Module: "two", Suite: "shared", Parameters: `{"case":2}`, Status: "fail"}, + } + model := buildReport(t.TempDir(), intake.Facts{Tests: tests}, false) + if model.Tests[0].Name == model.Tests[1].Name || model.Tests[0].Suite == model.Tests[1].Suite { + t.Fatal("table loses module or parameter identity") + } + report := renderTestReport(t, t.TempDir(), intake.Facts{Tests: tests}) + for _, want := range []string{`data-status="Passed"`, `data-status="Failed"`, `aria-controls="test-detail-0"`, `aria-controls="test-detail-1"`, `id="test-detail-0"`, `id="test-detail-1"`, `<script>alert(1)</script>`} { + if !strings.Contains(report, want) { + t.Errorf("report missing %q", want) + } + } + if strings.Contains(report, `same \nTest Suites: 1 failed, 1 total\n", events: 1, want: "Test Suites: 1 failed, 1 total"}, {name: "80 lines", output: strings.Repeat("log line\n", 79) + "final failure", events: 1, want: "final failure"}, {name: "long output", output: "initial failure\n" + strings.Repeat("log line\n", 79) + "final failure\n", events: 1, want: "... 1 line omitted; see the full test output below ..."}, } { @@ -399,6 +401,24 @@ func TestRunStillReportsEventsWhenJestFails(t *testing.T) { if string(contents) != tc.output { t.Fatalf("saved output changed: %q", contents) } + report, err := os.ReadFile(filepath.Join(installer.sessionDirectory, reportFilename)) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`
`, "Jest output", "npx jest", "Command failed: exit status 1", tc.output} { + if !strings.Contains(html.UnescapeString(string(report)), want) { + t.Errorf("report missing full command output or failure context %q", want) + } + } + if tc.output == "" && !strings.Contains(string(report), "The command produced no output.") { + t.Fatal("report hides empty command output") + } + if strings.Contains(string(report), "") { + t.Fatal("command output can execute HTML") + } + if strings.Contains(string(report), "
Test events
") { + t.Fatal("run details still shows the event count") + } if strings.Contains(output.String(), "Full test output: "+repositoryRoot) { t.Fatal("output path is absolute") } From d6c6acad2a8e81ef24fc4002b6748d3ee865eab9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 29 Sep 2026 13:48:46 +0200 Subject: [PATCH 13/26] Simplify report subtitle --- internal/testdrive/report.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index bca5162c..605dfc1c 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -1129,7 +1129,7 @@

Test report

- {{ .Runtime.Framework }} · local testdrive · {{ .Summary }} + {{ .Runtime.Framework }} · {{ .Summary }}

From d05d122164eae298255f6e29e6b351fccb4e5c11 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 29 Sep 2026 13:50:22 +0200 Subject: [PATCH 14/26] Omit raw request bodies from intake JSON artifacts --- internal/testdrive/intake/server.go | 4 ---- internal/testdrive/intake/server_test.go | 19 ++++++++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/testdrive/intake/server.go b/internal/testdrive/intake/server.go index 082a6a54..6acf0ebb 100644 --- a/internal/testdrive/intake/server.go +++ b/internal/testdrive/intake/server.go @@ -48,7 +48,6 @@ type storedRequest struct { Timestamp time.Time `json:"timestamp"` ContentEncoding string `json:"content_encoding,omitempty"` DecodeError string `json:"decode_error,omitempty"` - RawBody []byte `json:"raw_body,omitempty"` Method string `json:"method"` Path string `json:"path"` ContentType string `json:"content_type,omitempty"` @@ -170,9 +169,6 @@ func (s *Server) recordRequests(next http.Handler) http.Handler { } if decodeErr != nil { stored.DecodeError = decodeErr.Error() - stored.RawBody = rawRequest.Body - } else if !utf8.Valid(body) { - stored.RawBody = rawRequest.Body } s.requestsMu.Lock() persistErr := s.persistRequest(len(s.requests)+1, stored) diff --git a/internal/testdrive/intake/server_test.go b/internal/testdrive/intake/server_test.go index b804ac42..a17c00dc 100644 --- a/internal/testdrive/intake/server_test.go +++ b/internal/testdrive/intake/server_test.go @@ -152,6 +152,7 @@ func TestServerStoresMessagePackAsJSON(t *testing.T) { require.NoError(t, err) require.True(t, json.Valid(storedBytes)) require.Contains(t, string(storedBytes), `"events": []`) + require.NotContains(t, string(storedBytes), `"raw_body"`) } func TestServerStoresAndRecognizesGzippedMessagePack(t *testing.T) { @@ -240,6 +241,19 @@ func TestDecodeMultipartStoresEveryPartAsJSON(t *testing.T) { require.Equal(t, "events.msgpack", stored.Parts[1].Filename) require.JSONEq(t, `{"count":2}`, string(stored.Parts[1].Body)) require.JSONEq(t, `"hello"`, string(stored.Parts[2].Body)) + + server := &Server{directory: t.TempDir()} + request := httptest.NewRequest(http.MethodPost, "/observed", bytes.NewReader(body.Bytes())) + request.Header.Set("Content-Type", writer.FormDataContentType()) + response := httptest.NewRecorder() + server.recordRequests(newHandler()).ServeHTTP(response, request) + require.Equal(t, http.StatusOK, response.Code) + data, err := os.ReadFile(filepath.Join(server.directory, "001-request.json")) + require.NoError(t, err) + require.NotContains(t, string(data), `"raw_body"`) + var persisted storedRequest + require.NoError(t, json.Unmarshal(data, &persisted)) + require.JSONEq(t, string(decoded), string(persisted.Body)) } func TestRequestDecodingRejectsMalformedPayloads(t *testing.T) { @@ -334,7 +348,7 @@ func TestGzippedSettingsRequest(t *testing.T) { require.JSONEq(t, string(payload), string(stored.Body)) } -func TestFailedAndBinaryRequestsRemainOnDisk(t *testing.T) { +func TestStoredRequestsOmitRawBody(t *testing.T) { for _, tc := range []struct { name, contentType, encoding string body []byte @@ -365,13 +379,12 @@ func TestFailedAndBinaryRequestsRemainOnDisk(t *testing.T) { require.NoError(t, err) var stored struct { Path string `json:"path"` - RawBody []byte `json:"raw_body"` DecodeError string `json:"decode_error"` ContentEncoding string `json:"content_encoding"` } require.NoError(t, json.Unmarshal(data, &stored)) require.Equal(t, "/observed", stored.Path) - require.Equal(t, tc.body, stored.RawBody) + require.NotContains(t, string(data), `"raw_body"`) require.Equal(t, tc.encoding, stored.ContentEncoding) if tc.decodeError != "" { require.Contains(t, stored.DecodeError, tc.decodeError) From f1dcdb7d1b4e13ff248c0d35f9f972a7b507af80 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 29 Sep 2026 13:52:44 +0200 Subject: [PATCH 15/26] Show report coverage columns only at the reported level --- internal/testdrive/report.go | 28 ++++++++++++----------- internal/testdrive/report.html | 12 ++++++++-- internal/testdrive/report_test.go | 38 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index 5f5d69c3..fd22496e 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -110,15 +110,16 @@ type reportArtifact struct { } type reportModel struct { - Headline string - HeadlineTone string - Runtime reportRuntime - Summary string - Cards []reportCard - Facts []reportFact - Artifacts []reportArtifact - Suites []reportSuite - Tests []reportTest + Headline string + HeadlineTone string + CoverageLevel string + Runtime reportRuntime + Summary string + Cards []reportCard + Facts []reportFact + Artifacts []reportArtifact + Suites []reportSuite + Tests []reportTest } type reportRuntime struct{ Framework, Tracer, Command, Output, Error string } @@ -178,10 +179,11 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo showTestCoverage := findings.CoverageLevel == "test" showSuiteCoverage := findings.CoverageLevel == "suite" model := reportModel{ - Headline: fmt.Sprintf("%d test %s received.", findings.TestEventCount, plural(findings.TestEventCount, "event", "events")), - Summary: "No findings.", - HeadlineTone: factTone(findings.TestEventCount > 0), - Runtime: info, + Headline: fmt.Sprintf("%d test %s received.", findings.TestEventCount, plural(findings.TestEventCount, "event", "events")), + Summary: "No findings.", + HeadlineTone: factTone(findings.TestEventCount > 0), + Runtime: info, + CoverageLevel: findings.CoverageLevel, Facts: []reportFact{ {Label: "Tests with coverage", Value: fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount), Tone: factTone(findings.TestCount > 0 && findings.CoveredTestCount == findings.TestCount)}, {Label: info.Framework, Value: status, Tone: factTone(!commandFailed && findings.TestEventCount > 0)}, diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index 605dfc1c..5c0893d6 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -1361,7 +1361,9 @@

Run details

Suite Tests Duration + {{ if eq .CoverageLevel "suite" }} Suite coverage + {{ end }} Details @@ -1383,6 +1385,7 @@

Run details

{{ .TestCount }} {{ .Duration }} + {{ if eq $.CoverageLevel "suite" }} {{ if .ShowCoverage }} {{ len .CoveredFiles }} files @@ -1390,6 +1393,7 @@

Run details

Not reported {{ end }} + {{ end }} {{ template "icon-chevron-down" }}Run details - +
{{ if .ShowCoverage }}
@@ -1463,7 +1467,9 @@

Run details

File / suite Duration Attempts + {{ if eq .CoverageLevel "test" }} Test coverage + {{ end }} Details @@ -1490,6 +1496,7 @@

Run details

{{ .Duration }} {{ len .Attempts }} + {{ if eq $.CoverageLevel "test" }} {{ if .CoverageLevel }} {{ len .CoveredFiles }} files @@ -1497,6 +1504,7 @@

Run details

Not reported {{ end }} + {{ end }} {{ template "icon-chevron-down" }}Run details - + {{ template "test-detail" . }} diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index 1f7690df..e0fd9119 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -7,6 +7,7 @@ package testdrive import ( "bytes" + "fmt" "os" "path/filepath" "strings" @@ -559,3 +560,40 @@ func TestReportRunDetailsAlwaysVisibleAfterFindings(t *testing.T) { } } } + +func TestReportCoverageColumnsMatchReportedLevel(t *testing.T) { + for _, level := range []string{"test", "suite", ""} { + t.Run("level_"+level, func(t *testing.T) { + report := renderTestReport(t, t.TempDir(), intake.Facts{ + CoverageLevel: level, + Tests: []intake.Test{{Name: "one", Suite: "suite", Status: "pass"}}, + }) + for _, view := range []struct { + id, level string + columns int + }{{"tests", "test", 7}, {"suites", "suite", 6}} { + _, section, found := strings.Cut(report, `
") + showCoverage := level == view.level + if strings.Contains(section, `class="coverage-column"`) != showCoverage { + t.Fatalf("%s coverage column does not match level %q", view.id, level) + } + columns := view.columns + if showCoverage { + columns++ + } + _, row, _ := strings.Cut(section, ``) + row, _, _ = strings.Cut(row, "") + if strings.Count(row, " Date: Tue, 29 Sep 2026 13:54:37 +0200 Subject: [PATCH 16/26] Count suites in suite-level coverage summary --- internal/testdrive/report.go | 18 +++++++++++++++--- internal/testdrive/report_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index fd22496e..792ac172 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -178,6 +178,18 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo } showTestCoverage := findings.CoverageLevel == "test" showSuiteCoverage := findings.CoverageLevel == "suite" + suites := builder.reportSuites(findings.Tests, findings.SuiteCoverages, showSuiteCoverage) + coverageLabel := "Tests with coverage" + covered, total := findings.CoveredTestCount, findings.TestCount + if showSuiteCoverage { + coverageLabel = "Suites with coverage" + covered, total = 0, len(suites) + for _, suite := range suites { + if suite.CoveredCount > 0 { + covered++ + } + } + } model := reportModel{ Headline: fmt.Sprintf("%d test %s received.", findings.TestEventCount, plural(findings.TestEventCount, "event", "events")), Summary: "No findings.", @@ -185,7 +197,7 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo Runtime: info, CoverageLevel: findings.CoverageLevel, Facts: []reportFact{ - {Label: "Tests with coverage", Value: fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount), Tone: factTone(findings.TestCount > 0 && findings.CoveredTestCount == findings.TestCount)}, + {Label: coverageLabel, Value: fmt.Sprintf("%d / %d", covered, total), Tone: factTone(total > 0 && covered == total)}, {Label: info.Framework, Value: status, Tone: factTone(!commandFailed && findings.TestEventCount > 0)}, {Label: "Datadog library", Value: info.Tracer, Tone: "good"}, }, @@ -195,8 +207,8 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo }, Tests: builder.reportTests(findings.Tests, showTestCoverage), } - model.Suites = builder.reportSuites(findings.Tests, findings.SuiteCoverages, showSuiteCoverage) - if findings.CoveredTestCount == 0 { + model.Suites = suites + if covered == 0 { model.Facts[0].Value = "Not reported" } diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index e0fd9119..a7be9ed8 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -597,3 +597,29 @@ func TestReportCoverageColumnsMatchReportedLevel(t *testing.T) { }) } } + +func TestReportCoverageSummaryCountsTheReportedLevel(t *testing.T) { + tests := []intake.Test{ + {Module: "one", Suite: "shared", Name: "first"}, + {Module: "one", Suite: "shared", Name: "second"}, + {Module: "two", Suite: "shared", Name: "third"}, + } + for _, tc := range []struct { + name, level, label, value, tone string + coverages []intake.SuiteCoverage + }{ + {"test", "test", "Tests with coverage", "2 / 3", "attention", nil}, + {"partial suites", "suite", "Suites with coverage", "1 / 2", "attention", []intake.SuiteCoverage{{Module: "one", Suite: "shared", CoveredTests: 2}}}, + {"all suites", "suite", "Suites with coverage", "2 / 2", "good", []intake.SuiteCoverage{{Module: "one", Suite: "shared", CoveredTests: 2}, {Module: "two", Suite: "shared", CoveredTests: 1}}}, + {"no suite coverage", "suite", "Suites with coverage", "Not reported", "attention", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + facts := intake.Facts{CoverageLevel: tc.level, Tests: tests, TestCount: 3, CoveredTestCount: 2, SuiteCoverages: tc.coverages} + model := buildReport(t.TempDir(), facts, false) + want := reportFact{Label: tc.label, Value: tc.value, Tone: tc.tone} + if model.Facts[0] != want { + t.Fatalf("coverage summary = %+v, want %+v", model.Facts[0], want) + } + }) + } +} From 1ea634ce48294173b8da240d4a6129b93ce6fca8 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 29 Sep 2026 14:03:04 +0200 Subject: [PATCH 17/26] Move report totals into run details and drop event counts --- internal/testdrive/report.go | 7 ++--- internal/testdrive/report.html | 26 ++++------------ internal/testdrive/report_test.go | 45 +++++++++++++++------------- internal/testdrive/testdrive_test.go | 1 - 4 files changed, 31 insertions(+), 48 deletions(-) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index 792ac172..d36124b9 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -110,8 +110,7 @@ type reportArtifact struct { } type reportModel struct { - Headline string - HeadlineTone string + NoTestEvents bool CoverageLevel string Runtime reportRuntime Summary string @@ -191,9 +190,8 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo } } model := reportModel{ - Headline: fmt.Sprintf("%d test %s received.", findings.TestEventCount, plural(findings.TestEventCount, "event", "events")), Summary: "No findings.", - HeadlineTone: factTone(findings.TestEventCount > 0), + NoTestEvents: findings.TestEventCount == 0, Runtime: info, CoverageLevel: findings.CoverageLevel, Facts: []reportFact{ @@ -213,7 +211,6 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo } if findings.TestEventCount == 0 { - model.Headline = "No test events received." model.Summary = "Check the instrumentation setup." } if findings.EmptyCoverageEntryCount > 0 { diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index 5c0893d6..618695bd 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -467,22 +467,11 @@

align-items: center; gap: 26px; } - .run-summary { - border-left: 1px solid var(--line); - padding-left: 24px; - } - .run-summary strong { - display: block; + .run-totals { font-weight: 500; - margin-bottom: 4px; - } - .run-summary span { - color: var(--muted); - font-size: 13px; - } - .run-summary .attention { - color: var(--red); + margin: 12px 0 0; } + .no-test-events { color: var(--red); } .artifacts { display: flex; gap: 8px; @@ -1061,9 +1050,6 @@

.header-actions { gap: 14px; } - .run-summary { - display: none; - } .results-table { min-width: 850px; } @@ -1133,10 +1119,6 @@

Test report

-
- {{ len .Tests }} tests · {{ len .Suites }} suites{{ .Headline }} -
{{ range .Artifacts }} {{ .Title }} · {{ .Count }} aria-label="Run details" >

Run details

+

{{ len .Tests }} {{ plural (len .Tests) "test" "tests" }} · {{ len .Suites }} {{ plural (len .Suites) "suite" "suites" }}

+ {{ if .NoTestEvents }}

No test events received.

{{ end }}
{{ range .Facts }}
diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index a7be9ed8..cb3a296a 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -389,8 +389,8 @@ func TestBuildReportShowsNoEventsAndRollsUpUnknownSuite(t *testing.T) { {Name: "failed", Status: "fail", Duration: 2 * time.Millisecond}, } model := buildReport(t.TempDir(), intake.Facts{Tests: tests}, true) - if model.Headline != "No test events received." || model.Summary != "Check the instrumentation setup." { - t.Fatalf("headline = %q, summary = %q", model.Headline, model.Summary) + if !model.NoTestEvents || model.Summary != "Check the instrumentation setup." { + t.Fatalf("no events = %v, summary = %q", model.NoTestEvents, model.Summary) } if len(model.Suites) != 1 || model.Suites[0].Name != "Unknown suite" || model.Suites[0].Status != "Failed" { t.Fatalf("suite rollup = %#v", model.Suites) @@ -413,8 +413,8 @@ func renderTestReport(t *testing.T, repositoryRoot string, findings intake.Facts func TestReportSurfacesConfigurationErrorsDespiteReceivedTests(t *testing.T) { model := buildReport(t.TempDir(), intake.Facts{TestEventCount: 1, FailedTests: []intake.Test{{Name: "fails"}}, ConfigurationErrors: []string{"skippable_tests"}}, false) - if model.Headline != "1 test event received." { - t.Fatalf("headline overstates verification: %s", model.Headline) + if model.NoTestEvents { + t.Fatal("received events must not show the missing-events warning") } if !strings.Contains(model.Summary, "Tracer configuration errors: skippable_tests.") { t.Fatalf("missing configuration error: %s", model.Summary) @@ -462,7 +462,7 @@ func TestReportRuntimeFacts(t *testing.T) { if facts["Jest"] != tc.wantStatus || facts["Datadog library"] != tc.tracer { t.Fatalf("incorrect runtime facts: %v", facts) } - if tc.events == 0 && (model.Headline != "No test events received." || strings.Contains(model.Summary, "No findings.")) { + if tc.events == 0 && (!model.NoTestEvents || strings.Contains(model.Summary, "No findings.")) { t.Fatalf("report implies successful instrumentation without events: %+v", model) } }) @@ -518,22 +518,25 @@ func TestReportStatusFiltersDistinguishFlakySkippedAndUnknown(t *testing.T) { } } -func TestReportEventSummary(t *testing.T) { - for _, tc := range []struct { - name string - events int - want string - }{ - {"none", 0, `No test events received.`}, - {"single", 1, `1 test event received.`}, - {"multiple", 3, `3 test events received.`}, - } { - t.Run(tc.name, func(t *testing.T) { - report := renderTestReport(t, t.TempDir(), intake.Facts{TestEventCount: tc.events, Tests: []intake.Test{{Name: "one test"}}}) - if !strings.Contains(report, tc.want) { - t.Fatalf("report missing event summary %q", tc.want) - } - }) +func TestReportTotalsAppearOnlyInRunDetails(t *testing.T) { + for _, events := range []int{0, 1, 1363} { + report := renderTestReport(t, t.TempDir(), intake.Facts{TestEventCount: events, Tests: []intake.Test{{Name: "one test"}}}) + _, header, _ := strings.Cut(report, `") + if strings.Contains(header, "run-totals") || strings.Contains(header, "received") { + t.Fatal("header must not show totals or event messages") + } + _, details, _ := strings.Cut(report, `id="run-details"`) + details, _, _ = strings.Cut(details, "") + if !strings.Contains(details, `

1 test · 1 suite

`) { + t.Fatal("run details missing test and suite totals") + } + if strings.Contains(report, "1363") || strings.Contains(report, "test event received") || strings.Contains(report, "test events received") && events > 0 { + t.Fatal("report still displays received event counts") + } + if strings.Contains(details, `

No test events received.

`) != (events == 0) { + t.Fatal("missing-events warning must match event availability") + } } } diff --git a/internal/testdrive/testdrive_test.go b/internal/testdrive/testdrive_test.go index 9cc14bb4..7ec229f8 100644 --- a/internal/testdrive/testdrive_test.go +++ b/internal/testdrive/testdrive_test.go @@ -293,7 +293,6 @@ func TestRunReportsCapturedTestsAndCoverage(t *testing.T) { t.Fatal(err) } for _, expected := range []string{ - "2 test events received.", "Slow tests", "Median test time · 1s", "slow test", From 21ec76c7e8d8186a781713f01c5998b267cbf96d Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 29 Sep 2026 14:07:59 +0200 Subject: [PATCH 18/26] Add slow-suite findings and lower slow-test duration floor --- internal/testdrive/intake/findings.go | 57 ++++++++++++++++- internal/testdrive/intake/findings_test.go | 63 ++++++++++++++++++- internal/testdrive/report.go | 19 +++++- internal/testdrive/report.html | 72 +++++++++++++--------- internal/testdrive/report_test.go | 29 +++++++++ internal/testdrive/testdrive.go | 16 ++++- 6 files changed, 219 insertions(+), 37 deletions(-) diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go index ee49f224..56256dd8 100644 --- a/internal/testdrive/intake/findings.go +++ b/internal/testdrive/intake/findings.go @@ -19,7 +19,8 @@ import ( ) const ( - minimumSlowDuration = 5 * time.Second + minimumSlowTestDuration = time.Second + minimumSlowSuiteDuration = 5 * time.Second minimumBroadCoverageFiles = 5 ) @@ -69,6 +70,13 @@ type SuiteCoverage struct { CoveredTests int } +// SlowSuite describes a suite whose combined test duration is unusually high. +type SlowSuite struct { + Module string + Suite string + Duration time.Duration +} + // Facts contains the facts shown in the testdrive report. type Facts struct { ConfigurationErrors []string @@ -77,6 +85,7 @@ type Facts struct { TestEventCount int CoveredTestCount int TestDurationMedian time.Duration + SuiteDurationMedian time.Duration CoveredFilesMedian int CoverageLevel string SuiteCoverages []SuiteCoverage @@ -84,6 +93,7 @@ type Facts struct { FailedTests []Test FlakyTests []Test SlowTests []Test + SlowSuites []SlowSuite BroadCoverage []CoverageFact } @@ -103,6 +113,7 @@ func (s *Server) Facts() (Facts, error) { if findings.CoverageLevel == "suite" { findings.SuiteCoverages = suiteCoverages(tests, coverages) } + findings.SlowSuites, findings.SuiteDurationMedian = slowSuites(findings.Tests) findings.TestCount = len(findings.Tests) findings.CoveredTestCount = uniqueCoveredTestCount(tests, coverages) findings.BroadCoverage, findings.CoveredFilesMedian = analyzeCoverage(tests, coverages, findings.CoverageLevel) @@ -272,7 +283,7 @@ func slowTests(tests []Test) ([]Test, time.Duration) { slow := make([]Test, 0) for _, test := range tests { - if test.Duration > minimumSlowDuration && test.Duration >= median*5 { + if test.Duration > minimumSlowTestDuration && test.Duration >= median*5 { slow = append(slow, test) } } @@ -280,6 +291,41 @@ func slowTests(tests []Test) ([]Test, time.Duration) { return slow, median } +func slowSuites(tests []Test) ([]SlowSuite, time.Duration) { + bySuite := make(map[string]SlowSuite) + for _, test := range tests { + key := test.Module + "\x00" + test.Suite + suite := bySuite[key] + suite.Module, suite.Suite = test.Module, test.Suite + suite.Duration += test.Duration + bySuite[key] = suite + } + durations := make([]time.Duration, 0, len(bySuite)) + for _, suite := range bySuite { + durations = append(durations, suite.Duration) + } + median := medianDurations(durations) + var slow []SlowSuite + if len(bySuite) < 2 { + return slow, median + } + for _, suite := range bySuite { + if suite.Duration > minimumSlowSuiteDuration && suite.Duration >= median*5 { + slow = append(slow, suite) + } + } + sort.Slice(slow, func(i, j int) bool { + if slow[i].Duration != slow[j].Duration { + return slow[i].Duration > slow[j].Duration + } + if slow[i].Module != slow[j].Module { + return slow[i].Module < slow[j].Module + } + return slow[i].Suite < slow[j].Suite + }) + return slow, median +} + func medianTestDuration(tests []Test) time.Duration { if len(tests) == 0 { return 0 @@ -288,6 +334,13 @@ func medianTestDuration(tests []Test) time.Duration { for _, test := range tests { durations = append(durations, test.Duration) } + return medianDurations(durations) +} + +func medianDurations(durations []time.Duration) time.Duration { + if len(durations) == 0 { + return 0 + } sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) middle := len(durations) / 2 if len(durations)%2 == 1 { diff --git a/internal/testdrive/intake/findings_test.go b/internal/testdrive/intake/findings_test.go index de0645f3..0137213d 100644 --- a/internal/testdrive/intake/findings_test.go +++ b/internal/testdrive/intake/findings_test.go @@ -54,8 +54,8 @@ func TestAnalyzeTestsSlowThresholds(t *testing.T) { wantSlow bool }{ {name: "fast outlier", median: 10 * time.Millisecond, duration: 400 * time.Millisecond}, - {name: "exactly five seconds", median: time.Second, duration: 5 * time.Second}, - {name: "above five seconds", median: time.Second, duration: 5*time.Second + time.Nanosecond, wantSlow: true}, + {name: "exactly one second", median: 100 * time.Millisecond, duration: time.Second}, + {name: "above one second", median: 100 * time.Millisecond, duration: time.Second + time.Nanosecond, wantSlow: true}, {name: "below five times median", median: 2 * time.Second, duration: 10*time.Second - time.Nanosecond}, {name: "exactly five times median", median: 2 * time.Second, duration: 10 * time.Second, wantSlow: true}, {name: "above five times median", median: 2 * time.Second, duration: 11 * time.Second, wantSlow: true}, @@ -377,3 +377,62 @@ func TestAnalyzeTestsDoesNotUseSourceLocationAsIdentity(t *testing.T) { require.Len(t, flaky[0].Attempts, 2) require.Empty(t, failed) } + +func TestSlowSuiteThresholds(t *testing.T) { + for _, tc := range []struct { + name string + median, duration time.Duration + want bool + }{ + {"below floor", time.Millisecond, 4 * time.Second, false}, + {"at floor", time.Second, 5 * time.Second, false}, + {"above floor", time.Second, 5*time.Second + time.Nanosecond, true}, + {"below ratio", 2 * time.Second, 10*time.Second - time.Nanosecond, false}, + {"at ratio", 2 * time.Second, 10 * time.Second, true}, + {"uniform", 6 * time.Second, 6 * time.Second, false}, + } { + t.Run(tc.name, func(t *testing.T) { + slow, median := slowSuites([]Test{ + {Suite: "one", Duration: tc.median}, {Suite: "two", Duration: tc.median}, + {Suite: "candidate", Duration: tc.duration}, + }) + require.Equal(t, tc.median, median) + if tc.want { + require.Equal(t, []SlowSuite{{Suite: "candidate", Duration: tc.duration}}, slow) + } else { + require.Empty(t, slow) + } + }) + } + for _, tests := range [][]Test{nil, {{Suite: "only", Duration: time.Minute}}} { + slow, _ := slowSuites(tests) + require.Empty(t, slow) + } +} + +func TestFactsFindSlowSuitesWithoutCountingRetriesTwice(t *testing.T) { + var events []any + for i, tc := range []struct { + module, suite, name string + duration time.Duration + retry bool + }{ + {"one", "shared", "baseline", time.Second, false}, + {"two", "shared", "baseline", time.Second, false}, + {"three", "shared", "first", 4 * time.Second, false}, + {"three", "shared", "first", 20 * time.Second, true}, + {"three", "shared", "second", 2 * time.Second, false}, + } { + events = append(events, map[string]any{"type": "test", "content": map[string]any{ + "span_id": i + 1, "duration": int64(tc.duration), "meta": map[string]any{ + "test.module": tc.module, "test.suite": tc.suite, "test.name": tc.name, "test.status": "pass", "test.is_retry": tc.retry, + }, + }}) + } + payload, err := msgp.AppendIntf(nil, map[string]any{"events": events}) + require.NoError(t, err) + facts, err := serverWithCoverage(t, payload).Facts() + require.NoError(t, err) + require.Equal(t, time.Second, facts.SuiteDurationMedian) + require.Equal(t, []SlowSuite{{Module: "three", Suite: "shared", Duration: 6 * time.Second}}, facts.SlowSuites) +} diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index d36124b9..e51b8ffa 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -29,6 +29,7 @@ type reportCard struct { Count int Context string Tests []reportTest + Suites []reportSuite Coverages []reportCoverage } @@ -82,6 +83,7 @@ type reportSourceLine struct { } type reportSuite struct { + Key string Name string Status string Duration string @@ -240,6 +242,19 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo Tests: builder.reportTests(findings.SlowTests, showTestCoverage), }) } + if len(findings.SlowSuites) > 0 { + byKey := make(map[string]reportSuite, len(model.Suites)) + for _, suite := range model.Suites { + byKey[suite.Key] = suite + } + card := reportCard{Kind: "slow", Title: "Slow suites", Count: len(findings.SlowSuites), Context: "Median suite time · " + formatDuration(findings.SuiteDurationMedian)} + for _, suite := range findings.SlowSuites { + if row, ok := byKey[suite.Module+"\x00"+suite.Suite]; ok { + card.Suites = append(card.Suites, row) + } + } + model.Cards = append(model.Cards, card) + } if len(findings.BroadCoverage) > 0 { model.Cards = append(model.Cards, reportCard{ Kind: "coverage", Title: "Broad coverage", Count: len(findings.BroadCoverage), @@ -249,7 +264,7 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo } count := len(findings.ConfigurationErrors) + findings.EmptyCoverageEntryCount for _, size := range []int{ - len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.BroadCoverage), + len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.SlowSuites), len(findings.BroadCoverage), } { count += size } @@ -343,7 +358,7 @@ func (builder *reportBuilder) reportSuites(tests []intake.Test, coverages []inta if test.Module != "" { name = test.Module + " › " + name } - suite = &reportSuite{Name: name, Status: suiteStatus(status), ShowCoverage: showCoverage} + suite = &reportSuite{Key: key, Name: name, Status: suiteStatus(status), ShowCoverage: showCoverage} byName[key] = suite } else if suiteStatusRank(status) > suiteStatusRank(suite.Status) { suite.Status = suiteStatus(status) diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index 618695bd..11612c43 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -1,3 +1,35 @@ +{{ define "suite-detail" }} +
+ {{ if .ShowCoverage }} +
+ + Suite coverage · {{ len .CoveredFiles }} + {{ plural (len .CoveredFiles) "file" "files" }} · + {{ .CoveredCount }} tests with coverage + + {{ template "covered-files" .CoveredFiles }} +
+ {{ end }} + + + + + + + + + + {{ range .Tests }} + + + + + + {{ end }} + +
TestStatusDuration
{{ .Name }}{{ template "status" .Status }}{{ .Duration }}
+
+{{ end }}