{{.Title}}
{{.Context}}
- {{range .Tests}}
- {{.Suite}} · {{.Name}} {{end}}{{range .Coverages}}
- {{.Name}} · {{.FileCount}} files {{end}}
diff --git a/internal/testdrive/format.go b/internal/testdrive/format.go deleted file mode 100644 index a315d2fd..00000000 --- a/internal/testdrive/format.go +++ /dev/null @@ -1,66 +0,0 @@ -package testdrive - -import ( - "github.com/DataDog/ddtest/internal/testdrive/intake" - "strings" - "time" -) - -func testDisplayStatus(test intake.Test) (string, string) { - status := test.Status - sawPass := status == "pass" - sawFailure := status == "fail" - for _, attempt := range test.Attempts { - sawPass = sawPass || attempt.Status == "pass" - sawFailure = sawFailure || attempt.Status == "fail" - } - if sawPass && sawFailure { - return "Flaky", "attention" - } - if status == "" && len(test.Attempts) > 0 { - status = test.Attempts[len(test.Attempts)-1].Status - } - return displayStatus(status), attemptTone(status) -} - -func attemptTone(status string) string { - if status == "pass" { - return "good" - } - return "attention" -} - -func findingDuration(test intake.Test) time.Duration { - if test.Duration != 0 || len(test.Attempts) == 0 { - return test.Duration - } - return test.Attempts[0].Duration -} - -func passedFailed(passed bool) string { - if passed { - return "Passed" - } - return "Failed" -} - -func displayStatus(status string) string { - if status == "" { - return "Unknown" - } - return strings.ToUpper(status[:1]) + status[1:] -} - -func formatDuration(duration time.Duration) string { - if duration < time.Millisecond { - return duration.Round(time.Microsecond).String() - } - return duration.Round(time.Millisecond).String() -} - -func plural(count int, singular, plural string) string { - if count == 1 { - return singular - } - return plural -} diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go index a95fa68b..1a5a7252 100644 --- a/internal/testdrive/intake/findings.go +++ b/internal/testdrive/intake/findings.go @@ -19,7 +19,7 @@ import ( ) const ( - minimumSlowDuration = 100 * time.Millisecond + minimumSlowDuration = 5 * time.Second minimumBroadCoverageFiles = 5 ) @@ -215,11 +215,10 @@ func slowTests(tests []Test) ([]Test, time.Duration) { return nil, medianTestDuration(tests) } median := medianTestDuration(tests) - threshold := max(minimumSlowDuration, median*2) slow := make([]Test, 0) for _, test := range tests { - if test.Duration >= threshold && test.Duration > median { + if test.Duration > minimumSlowDuration && test.Duration >= median*5 { slow = append(slow, test) } } diff --git a/internal/testdrive/intake/findings_test.go b/internal/testdrive/intake/findings_test.go index 80998472..45177c60 100644 --- a/internal/testdrive/intake/findings_test.go +++ b/internal/testdrive/intake/findings_test.go @@ -21,7 +21,7 @@ func TestAnalyzeTestsFindsFailuresRetriesAndSlowTests(t *testing.T) { {name: "fast", suite: "one.test.js", status: "pass", duration: 10 * time.Millisecond}, {name: "flaky", suite: "one.test.js", status: "fail", duration: 20 * time.Millisecond}, {name: "flaky", suite: "one.test.js", status: "pass", duration: 30 * time.Millisecond, isRetry: true}, - {name: "slow", suite: "two.test.js", status: "pass", duration: 400 * time.Millisecond}, + {name: "slow", suite: "two.test.js", status: "pass", duration: 6 * time.Second}, {name: "broken", suite: "two.test.js", status: "fail", duration: 15 * time.Millisecond}, } @@ -40,12 +40,44 @@ func TestAnalyzeTestsFindsFailuresRetriesAndSlowTests(t *testing.T) { }, }}, flaky) require.Equal(t, []Test{{ - Name: "slow", Suite: "two.test.js", Status: "pass", Duration: 400 * time.Millisecond, - Attempts: []TestRun{{Status: "pass", Duration: 400 * time.Millisecond}}, + Name: "slow", Suite: "two.test.js", Status: "pass", Duration: 6 * time.Second, + Attempts: []TestRun{{Status: "pass", Duration: 6 * time.Second}}, }}, slow) require.Equal(t, 17500*time.Microsecond, median) } +func TestAnalyzeTestsSlowThresholds(t *testing.T) { + for _, tc := range []struct { + name string + median time.Duration + duration time.Duration + wantSlow bool + }{ + {name: "fast outlier", median: 10 * time.Millisecond, duration: 400 * time.Millisecond}, + {name: "exactly five seconds", median: time.Second, duration: 5 * time.Second}, + {name: "above five seconds", median: time.Second, duration: 5*time.Second + time.Nanosecond, wantSlow: true}, + {name: "below five times median", median: 2 * time.Second, duration: 10*time.Second - time.Nanosecond}, + {name: "exactly five times median", median: 2 * time.Second, duration: 10 * time.Second, wantSlow: true}, + {name: "above five times median", median: 2 * time.Second, duration: 11 * time.Second, wantSlow: true}, + {name: "uniformly slow tests", median: 6 * time.Second, duration: 6 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, _, slow, median := analyzeTests([]testReference{ + {name: "baseline one", status: "pass", duration: tc.median}, + {name: "baseline two", status: "pass", duration: tc.median}, + {name: "candidate", status: "pass", duration: tc.duration}, + }, nil, "") + require.Equal(t, tc.median, median) + if tc.wantSlow { + require.Len(t, slow, 1) + require.Equal(t, "candidate", slow[0].Name) + } else { + require.Empty(t, slow) + } + }) + } +} + func TestAnalyzeTestsUsesFinalStatusAndMarksMixedOutcomesFlaky(t *testing.T) { tests := []testReference{ {name: "flaky", suite: "one.test.js", status: "pass", finalStatus: "pass", duration: 10 * time.Millisecond}, @@ -250,7 +282,7 @@ func TestFindingsPreservesCoverageInEveryCategory(t *testing.T) { } duration := time.Millisecond if name == "" { - duration = time.Second + duration = 6 * time.Second } events = append(events, map[string]any{"type": "test", "content": map[string]any{ "test_session_id": 1, "test_suite_id": 2, "span_id": i + 1, "duration": int64(duration), diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go new file mode 100644 index 00000000..1887cdde --- /dev/null +++ b/internal/testdrive/report.go @@ -0,0 +1,192 @@ +package testdrive + +import ( + "errors" + "fmt" + "github.com/DataDog/ddtest/internal/testdrive/intake" + "html/template" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +const reportFilename = "report.html" + +type reportRuntime struct{ Framework, Tracer string } +type reportFact struct{ Label, Value string } +type reportCard struct { + Title, Context string + Tests []intake.Test + Coverages []intake.CoverageFact +} +type reportModel struct { + Headline, Summary string + Facts []reportFact + Cards []reportCard +} + +func writeReport(repositoryRoot, sessionDirectory string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) (string, error) { + path := filepath.Join(sessionDirectory, reportFilename) + file, err := os.Create(path) + if err != nil { + return "", fmt.Errorf("create testdrive report: %w", err) + } + executeErr := testdriveReport.Execute(file, buildReport(repositoryRoot, findings, commandFailed, runtime...)) + if err := errors.Join(executeErr, file.Close()); err != nil { + return "", fmt.Errorf("write testdrive report: %w", err) + } + return path, nil +} +func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) reportModel { + info := reportRuntime{Framework: "Test command", Tracer: "Isolated installation"} + if len(runtime) > 0 { + info = runtime[0] + } + coverage := "Not reported" + if findings.CoveredTestCount > 0 { + coverage = fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount) + } + status := passedFailed(!commandFailed) + if findings.TestEventCount == 0 { + status = "No test results received" + } + model := reportModel{Headline: "Test events received.", Summary: "No findings.", Facts: []reportFact{ + {"Test events", fmt.Sprint(findings.TestEventCount)}, + {"Tests with coverage", coverage}, + {info.Framework, status}, + {"Datadog library", info.Tracer}, + }} + if findings.TestEventCount == 0 { + model.Headline = "No test events received." + model.Summary = "Check the instrumentation setup." + } + if findings.EmptyCoverageEntryCount > 0 { + model.Cards = append(model.Cards, reportCard{ + Title: "Tracer error: empty coverage entries", + Context: fmt.Sprintf("%d coverage entries had an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.", findings.EmptyCoverageEntryCount), + }) + } + if len(findings.FailedTests) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any tests failed?", Tests: findings.FailedTests}) + } + if len(findings.FlakyTests) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any flaky tests?", Tests: findings.FlakyTests}) + } + if len(findings.SlowTests) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any tests slower than the others?", Context: "Median test time · " + formatDuration(findings.TestDurationMedian), Tests: findings.SlowTests}) + } + if len(findings.BroadCoverage) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any unusually broad test coverage?", Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), Coverages: findings.BroadCoverage}) + } + count := len(findings.ConfigurationErrors) + findings.EmptyCoverageEntryCount + for _, size := range []int{ + len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.BroadCoverage), + } { + count += size + } + if count > 0 { + model.Summary = fmt.Sprintf("%d %s.", count, plural(count, "finding", "findings")) + } + if len(findings.ConfigurationErrors) > 0 { + model.Summary += " Tracer configuration errors: " + strings.Join(findings.ConfigurationErrors, ", ") + ". Inspect the captured traffic and test output." + } + return model +} + +var testdriveReport = template.Must(template.New("testdrive-report").Parse(` +
{{.Summary}}
+{{range .Cards}}{{.Context}}