[SDTEST-3895] Generate HTML testdrive reports

internal/testdrive/intake/findings.go

@@ -19,7 +19,7 @@ import (
 )
 
 const (
-	minimumSlowDuration       = 100 * time.Millisecond
+	minimumSlowDuration       = 5 * time.Second
 	minimumBroadCoverageFiles = 5
 )
 
@@ -215,11 +215,10 @@ func slowTests(tests []Test) ([]Test, time.Duration) {
 		return nil, medianTestDuration(tests)
 	}
 	median := medianTestDuration(tests)
-	threshold := max(minimumSlowDuration, median*2)
 
 	slow := make([]Test, 0)
 	for _, test := range tests {
-		if test.Duration >= threshold && test.Duration > median {
+		if test.Duration > minimumSlowDuration && test.Duration >= median*5 {
 			slow = append(slow, test)
 		}
 	}

internal/testdrive/intake/findings_test.go

@@ -21,7 +21,7 @@ func TestAnalyzeTestsFindsFailuresRetriesAndSlowTests(t *testing.T) {
 		{name: "fast", suite: "one.test.js", status: "pass", duration: 10 * time.Millisecond},
 		{name: "flaky", suite: "one.test.js", status: "fail", duration: 20 * time.Millisecond},
 		{name: "flaky", suite: "one.test.js", status: "pass", duration: 30 * time.Millisecond, isRetry: true},
-		{name: "slow", suite: "two.test.js", status: "pass", duration: 400 * time.Millisecond},
+		{name: "slow", suite: "two.test.js", status: "pass", duration: 6 * time.Second},
 		{name: "broken", suite: "two.test.js", status: "fail", duration: 15 * time.Millisecond},
 	}
 
@@ -40,12 +40,44 @@ func TestAnalyzeTestsFindsFailuresRetriesAndSlowTests(t *testing.T) {
 		},
 	}}, flaky)
 	require.Equal(t, []Test{{
-		Name: "slow", Suite: "two.test.js", Status: "pass", Duration: 400 * time.Millisecond,
-		Attempts: []TestRun{{Status: "pass", Duration: 400 * time.Millisecond}},
+		Name: "slow", Suite: "two.test.js", Status: "pass", Duration: 6 * time.Second,
+		Attempts: []TestRun{{Status: "pass", Duration: 6 * time.Second}},
 	}}, slow)
 	require.Equal(t, 17500*time.Microsecond, median)
 }
 
+func TestAnalyzeTestsSlowThresholds(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		median   time.Duration
+		duration time.Duration
+		wantSlow bool
+	}{
+		{name: "fast outlier", median: 10 * time.Millisecond, duration: 400 * time.Millisecond},
+		{name: "exactly five seconds", median: time.Second, duration: 5 * time.Second},
+		{name: "above five seconds", median: time.Second, duration: 5*time.Second + time.Nanosecond, wantSlow: true},
+		{name: "below five times median", median: 2 * time.Second, duration: 10*time.Second - time.Nanosecond},
+		{name: "exactly five times median", median: 2 * time.Second, duration: 10 * time.Second, wantSlow: true},
+		{name: "above five times median", median: 2 * time.Second, duration: 11 * time.Second, wantSlow: true},
+		{name: "uniformly slow tests", median: 6 * time.Second, duration: 6 * time.Second},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			_, _, _, slow, median := analyzeTests([]testReference{
+				{name: "baseline one", status: "pass", duration: tc.median},
+				{name: "baseline two", status: "pass", duration: tc.median},
+				{name: "candidate", status: "pass", duration: tc.duration},
+			}, nil, "")
+			require.Equal(t, tc.median, median)
+			if tc.wantSlow {
+				require.Len(t, slow, 1)
+				require.Equal(t, "candidate", slow[0].Name)
+			} else {
+				require.Empty(t, slow)
+			}
+		})
+	}
+}
+
 func TestAnalyzeTestsUsesFinalStatusAndMarksMixedOutcomesFlaky(t *testing.T) {
 	tests := []testReference{
 		{name: "flaky", suite: "one.test.js", status: "pass", finalStatus: "pass", duration: 10 * time.Millisecond},
@@ -250,7 +282,7 @@ func TestFindingsPreservesCoverageInEveryCategory(t *testing.T) {
 				}
 				duration := time.Millisecond
 				if name == "" {
-					duration = time.Second
+					duration = 6 * time.Second
 				}
 				events = append(events, map[string]any{"type": "test", "content": map[string]any{
 					"test_session_id": 1, "test_suite_id": 2, "span_id": i + 1, "duration": int64(duration),

internal/testdrive/report.go

@@ -0,0 +1,192 @@
+package testdrive
+
+import (
+	"errors"
+	"fmt"
+	"github.com/DataDog/ddtest/internal/testdrive/intake"
+	"html/template"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+const reportFilename = "report.html"
+
+type reportRuntime struct{ Framework, Tracer string }
+type reportFact struct{ Label, Value string }
+type reportCard struct {
+	Title, Context string
+	Tests          []intake.Test
+	Coverages      []intake.CoverageFact
+}
+type reportModel struct {
+	Headline, Summary string
+	Facts             []reportFact
+	Cards             []reportCard
+}
+
+func writeReport(repositoryRoot, sessionDirectory string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) (string, error) {
+	path := filepath.Join(sessionDirectory, reportFilename)
+	file, err := os.Create(path)
+	if err != nil {
+		return "", fmt.Errorf("create testdrive report: %w", err)
+	}
+	executeErr := testdriveReport.Execute(file, buildReport(repositoryRoot, findings, commandFailed, runtime...))
+	if err := errors.Join(executeErr, file.Close()); err != nil {
+		return "", fmt.Errorf("write testdrive report: %w", err)
+	}
+	return path, nil
+}
+func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) reportModel {
+	info := reportRuntime{Framework: "Test command", Tracer: "Isolated installation"}
+	if len(runtime) > 0 {
+		info = runtime[0]
+	}
+	coverage := "Not reported"
+	if findings.CoveredTestCount > 0 {
+		coverage = fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount)
+	}
+	status := passedFailed(!commandFailed)
+	if findings.TestEventCount == 0 {
+		status = "No test results received"
+	}
+	model := reportModel{Headline: "Test events received.", Summary: "No findings.", Facts: []reportFact{
+		{"Test events", fmt.Sprint(findings.TestEventCount)},
+		{"Tests with coverage", coverage},
+		{info.Framework, status},
+		{"Datadog library", info.Tracer},
+	}}
+	if findings.TestEventCount == 0 {
+		model.Headline = "No test events received."
+		model.Summary = "Check the instrumentation setup."
+	}
+	if findings.EmptyCoverageEntryCount > 0 {
+		model.Cards = append(model.Cards, reportCard{
+			Title:   "Tracer error: empty coverage entries",
+			Context: fmt.Sprintf("%d coverage entries had an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.", findings.EmptyCoverageEntryCount),
+		})
+	}
+	if len(findings.FailedTests) > 0 {
+		model.Cards = append(model.Cards, reportCard{Title: "Any tests failed?", Tests: findings.FailedTests})
+	}
+	if len(findings.FlakyTests) > 0 {
+		model.Cards = append(model.Cards, reportCard{Title: "Any flaky tests?", Tests: findings.FlakyTests})
+	}
+	if len(findings.SlowTests) > 0 {
+		model.Cards = append(model.Cards, reportCard{Title: "Any tests slower than the others?", Context: "Median test time · " + formatDuration(findings.TestDurationMedian), Tests: findings.SlowTests})
+	}
+	if len(findings.BroadCoverage) > 0 {
+		model.Cards = append(model.Cards, reportCard{Title: "Any unusually broad test coverage?", Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), Coverages: findings.BroadCoverage})
+	}
+	count := len(findings.ConfigurationErrors) + findings.EmptyCoverageEntryCount
+	for _, size := range []int{
+		len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.BroadCoverage),
+	} {
+		count += size
+	}
+	if count > 0 {
+		model.Summary = fmt.Sprintf("%d %s.", count, plural(count, "finding", "findings"))
+	}
+	if len(findings.ConfigurationErrors) > 0 {
+		model.Summary += " Tracer configuration errors: " + strings.Join(findings.ConfigurationErrors, ", ") + ". Inspect the captured traffic and test output."
+	}
+	return model
+}
+
+var testdriveReport = template.Must(template.New("testdrive-report").Parse(`<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><title>DDTest report</title>
+<style>body{font:16px system-ui;max-width:900px;margin:3rem auto;padding:1rem}article,section{margin:2rem 0}li{margin:.5rem 0}</style></head><body>
+<h1>{{.Headline}}</h1><p>{{.Summary}}</p>
+{{range .Cards}}<article class="problem-card"><h2>{{.Title}}</h2><p>{{.Context}}</p><ul>{{range .Tests}}<li>{{.Suite}} · {{.Name}}</li>{{end}}{{range .Coverages}}<li>{{.Name}} · {{.FileCount}} files</li>{{end}}</ul></article>{{end}}
+<section><h2>Run details</h2><dl>{{range .Facts}}<dt>{{.Label}}</dt><dd>{{.Value}}</dd>{{end}}</dl></section>
+<section><h2>Artifacts</h2><ul><li><a href="intake/">JSON traffic</a></li><li><a href="test-output.txt">Test output</a></li></ul></section></body></html>`))
+
+func testDisplayStatus(test intake.Test) (string, string) {
+	status := test.Status
+	sawPass := status == "pass"
+	sawFailure := status == "fail"
+	for _, attempt := range test.Attempts {
+		sawPass = sawPass || attempt.Status == "pass"
+		sawFailure = sawFailure || attempt.Status == "fail"
+	}
+	if sawPass && sawFailure {
+		return "Flaky", "attention"
+	}
+	if status == "" && len(test.Attempts) > 0 {
+		status = test.Attempts[len(test.Attempts)-1].Status
+	}
+	return displayStatus(status), attemptTone(status)
+}
+
+func attemptTone(status string) string {
+	if status == "pass" {
+		return "good"
+	}
+	return "attention"
+}
+
+func findingDuration(test intake.Test) time.Duration {
+	if test.Duration != 0 || len(test.Attempts) == 0 {
+		return test.Duration
+	}
+	return test.Attempts[0].Duration
+}
+
+func passedFailed(passed bool) string {
+	if passed {
+		return "Passed"
+	}
+	return "Failed"
+}
+
+func displayStatus(status string) string {
+	if status == "" {
+		return "Unknown"
+	}
+	return strings.ToUpper(status[:1]) + status[1:]
+}
+
+func formatDuration(duration time.Duration) string {
+	if duration < time.Millisecond {
+		return duration.Round(time.Microsecond).String()
+	}
+	return duration.Round(time.Millisecond).String()
+}
+
+func plural(count int, singular, plural string) string {
+	if count == 1 {
+		return singular
+	}
+	return plural
+}
+
+func fileURL(path string) (string, error) {
+	absolutePath, err := filepath.Abs(path)
+	if err != nil {
+		return "", err
+	}
+	return absoluteFileURL(absolutePath), nil
+}
+
+func absoluteFileURL(absolutePath string) string {
+	slashPath := absolutePath
+	// A backslash is a valid filename character in an absolute Unix path.
+	if !strings.HasPrefix(absolutePath, "/") {
+		slashPath = strings.ReplaceAll(absolutePath, `\`, "/")
+	}
+	if strings.HasPrefix(absolutePath, `\\`) {
+		hostAndPath := strings.TrimPrefix(slashPath, "//")
+		host, path, _ := strings.Cut(hostAndPath, "/")
+		return (&url.URL{Scheme: "file", Host: host, Path: "/" + path}).String()
+	}
+	if len(slashPath) >= 2 && slashPath[1] == ':' {
+		slashPath = "/" + slashPath
+	}
+	return (&url.URL{Scheme: "file", Path: slashPath}).String()
+}
+
+func terminalLink(target, label string) string {
+	return "\x1b]8;;" + target + "\x1b\\" + label + "\x1b]8;;\x1b\\"
+}