From 67077ba7318e7fd82af7f0d427dab1ebab9b92fb Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:12:00 +0200 Subject: [PATCH 01/11] Generate linked HTML testdrive reports --- internal/testdrive/format.go | 66 ----------- internal/testdrive/report.go | 159 +++++++++++++++++++++++++++ internal/testdrive/report_test.go | 41 +++++++ internal/testdrive/testdrive_test.go | 31 +++++- 4 files changed, 229 insertions(+), 68 deletions(-) delete mode 100644 internal/testdrive/format.go create mode 100644 internal/testdrive/report.go create mode 100644 internal/testdrive/report_test.go diff --git a/internal/testdrive/format.go b/internal/testdrive/format.go deleted file mode 100644 index a315d2fd..00000000 --- a/internal/testdrive/format.go +++ /dev/null @@ -1,66 +0,0 @@ -package testdrive - -import ( - "github.com/DataDog/ddtest/internal/testdrive/intake" - "strings" - "time" -) - -func testDisplayStatus(test intake.Test) (string, string) { - status := test.Status - sawPass := status == "pass" - sawFailure := status == "fail" - for _, attempt := range test.Attempts { - sawPass = sawPass || attempt.Status == "pass" - sawFailure = sawFailure || attempt.Status == "fail" - } - if sawPass && sawFailure { - return "Flaky", "attention" - } - if status == "" && len(test.Attempts) > 0 { - status = test.Attempts[len(test.Attempts)-1].Status - } - return displayStatus(status), attemptTone(status) -} - -func attemptTone(status string) string { - if status == "pass" { - return "good" - } - return "attention" -} - -func findingDuration(test intake.Test) time.Duration { - if test.Duration != 0 || len(test.Attempts) == 0 { - return test.Duration - } - return test.Attempts[0].Duration -} - -func passedFailed(passed bool) string { - if passed { - return "Passed" - } - return "Failed" -} - -func displayStatus(status string) string { - if status == "" { - return "Unknown" - } - return strings.ToUpper(status[:1]) + status[1:] -} - -func formatDuration(duration time.Duration) string { - if duration < time.Millisecond { - return duration.Round(time.Microsecond).String() - } - return duration.Round(time.Millisecond).String() -} - -func plural(count int, singular, plural string) string { - if count == 1 { - return singular - } - return plural -} diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go new file mode 100644 index 00000000..70631544 --- /dev/null +++ b/internal/testdrive/report.go @@ -0,0 +1,159 @@ +package testdrive + +import ( + "errors" + "fmt" + "github.com/DataDog/ddtest/internal/testdrive/intake" + "html/template" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +const reportFilename = "report.html" + +type reportRuntime struct{ Framework, Tracer string } +type reportFact struct{ Label, Value string } +type reportCard struct { + Title, Context string + Tests []intake.TestFinding + Coverages []intake.CoverageFinding +} +type reportModel struct { + Headline, Summary string + Facts []reportFact + Cards []reportCard +} + +func writeReport(repositoryRoot, sessionDirectory string, findings intake.Findings, commandFailed bool, runtime ...reportRuntime) (string, error) { + path := filepath.Join(sessionDirectory, reportFilename) + file, err := os.Create(path) + if err != nil { + return "", fmt.Errorf("create testdrive report: %w", err) + } + executeErr := testdriveReport.Execute(file, buildReport(repositoryRoot, findings, commandFailed, runtime...)) + if err := errors.Join(executeErr, file.Close()); err != nil { + return "", fmt.Errorf("write testdrive report: %w", err) + } + return path, nil +} +func buildReport(_ string, findings intake.Findings, commandFailed bool, runtime ...reportRuntime) reportModel { + info := reportRuntime{Framework: "Test command", Tracer: "Isolated installation"} + if len(runtime) > 0 { + info = runtime[0] + } + coverage := "Not reported" + if findings.CoveredTestCount > 0 { + coverage = fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount) + } + model := reportModel{Headline: "Test events received.", Summary: "No findings.", Facts: []reportFact{ + {"Test events", fmt.Sprint(findings.TestEventCount)}, + {"Tests with coverage", coverage}, + {info.Framework, passedFailed(!commandFailed)}, + {"Tracer", info.Tracer + " · isolated"}, + }} + if findings.TestEventCount == 0 { + model.Headline = "No test events received." + model.Summary = "Check the instrumentation setup." + } + if len(findings.FailedTests) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any tests failed?", Tests: findings.FailedTests}) + } + if len(findings.FlakyTests) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any flaky tests?", Tests: findings.FlakyTests}) + } + if len(findings.SlowTests) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any tests slower than the others?", Context: "Median test time · " + formatDuration(findings.TestDurationMedian), Tests: findings.SlowTests}) + } + if len(findings.BroadCoverage) > 0 { + model.Cards = append(model.Cards, reportCard{Title: "Any unusually broad test coverage?", Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), Coverages: findings.BroadCoverage}) + } + if len(model.Cards) > 0 && findings.TestEventCount > 0 { + model.Summary = fmt.Sprintf("%d findings.", len(model.Cards)) + } + if len(findings.ConfigurationErrors) > 0 { + model.Summary = "Tracer configuration errors: " + strings.Join(findings.ConfigurationErrors, ", ") + ". Inspect the captured traffic and test output." + } + return model +} + +var testdriveReport = template.Must(template.New("testdrive-report").Parse(` +DDTest report + +

{{.Headline}}

{{.Summary}}

+{{range .Cards}}

{{.Title}}

{{.Context}}

{{end}} +

Run details

{{range .Facts}}
{{.Label}}
{{.Value}}
{{end}}
+

Artifacts

`)) + +func testDisplayStatus(test intake.TestFinding) (string, string) { + status := test.Status + sawPass := status == "pass" + sawFailure := status == "fail" + for _, attempt := range test.Attempts { + sawPass = sawPass || attempt.Status == "pass" + sawFailure = sawFailure || attempt.Status == "fail" + } + if sawPass && sawFailure { + return "Flaky", "attention" + } + if status == "" && len(test.Attempts) > 0 { + status = test.Attempts[len(test.Attempts)-1].Status + } + return displayStatus(status), attemptTone(status) +} + +func attemptTone(status string) string { + if status == "pass" { + return "good" + } + return "attention" +} + +func findingDuration(test intake.TestFinding) time.Duration { + if test.Duration != 0 || len(test.Attempts) == 0 { + return test.Duration + } + return test.Attempts[0].Duration +} + +func passedFailed(passed bool) string { + if passed { + return "Passed" + } + return "Failed" +} + +func displayStatus(status string) string { + if status == "" { + return "Unknown" + } + return strings.ToUpper(status[:1]) + status[1:] +} + +func formatDuration(duration time.Duration) string { + if duration < time.Millisecond { + return duration.Round(time.Microsecond).String() + } + return duration.Round(time.Millisecond).String() +} + +func plural(count int, singular, plural string) string { + if count == 1 { + return singular + } + return plural +} + +func fileURL(path string) (string, error) { + absolutePath, err := filepath.Abs(path) + if err != nil { + return "", err + } + return (&url.URL{Scheme: "file", Path: absolutePath}).String(), nil +} + +func terminalLink(target string) string { + return "\x1b]8;;" + target + "\x1b\\" + target + "\x1b]8;;\x1b\\" +} diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go new file mode 100644 index 00000000..63bc0d61 --- /dev/null +++ b/internal/testdrive/report_test.go @@ -0,0 +1,41 @@ +package testdrive + +import ( + "github.com/DataDog/ddtest/internal/testdrive/intake" + "os" + "strings" + "testing" +) + +func TestStaticReportEscapesFindingsAndDescribesMissingCoverage(t *testing.T) { + path, err := writeReport(t.TempDir(), t.TempDir(), intake.Findings{TestEventCount: 1, TestCount: 1, FailedTests: []intake.TestFinding{{Name: ""}}}, true) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"<script>", "Not reported", "Failed", "intake/", "test-output.txt"} { + if !strings.Contains(string(data), want) { + t.Errorf("missing %q", want) + } + } + if strings.Contains(string(data), ""}}}, true) + path, err := writeReport(t.TempDir(), t.TempDir(), intake.Facts{TestEventCount: 1, TestCount: 1, FailedTests: []intake.Test{{Name: ""}}}, true) if err != nil { t.Fatal(err) } @@ -24,14 +24,14 @@ func TestStaticReportEscapesFindingsAndDescribesMissingCoverage(t *testing.T) { if strings.Contains(string(data), ""}}}, true) if err != nil { From bf83efa48c7d806b2136d76b0ff14bef7a36c2eb Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 15:49:18 +0200 Subject: [PATCH 07/11] Simplify testdrive output and display relative paths --- internal/testdrive/report.go | 4 ++-- internal/testdrive/testdrive.go | 17 +++++++++++++---- internal/testdrive/testdrive_test.go | 21 +++++++++++++++++---- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index c25c93dd..7634ac3a 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -187,6 +187,6 @@ func absoluteFileURL(absolutePath string) string { return (&url.URL{Scheme: "file", Path: slashPath}).String() } -func terminalLink(target string) string { - return "\x1b]8;;" + target + "\x1b\\" + target + "\x1b]8;;\x1b\\" +func terminalLink(target, label string) string { + return "\x1b]8;;" + target + "\x1b\\" + label + "\x1b]8;;\x1b\\" } diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index cbef39cc..608e9460 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -115,6 +115,9 @@ func displayName(name string) string { func (t *Testdrive) Preview(output io.Writer) { command, args := t.command, t.args directory := t.session.Directory() + if relative, err := filepath.Rel(t.repositoryRoot, directory); err == nil { + directory = relative + } _, _ = fmt.Fprintf(output, "DDTest found %s and %s.\n", displayName(t.language), displayName(t.framework.Name())) _, _ = fmt.Fprintln(output) @@ -123,12 +126,14 @@ func (t *Testdrive) Preview(output io.Writer) { if t.projectTracer != "" { _, _ = fmt.Fprintf(output, " - reuse installed %s; no installation is needed\n", t.installedTracerLabel(t.projectTracer)) } else { - _, _ = fmt.Fprintf(output, " - install %s: %s\n", t.tracerLabel, shellquote.Join(append([]string{t.installCommand}, t.installArgs...)...)) + install := append([]string{t.installCommand}, t.installArgs...) + for i, arg := range install { + install[i] = strings.ReplaceAll(arg, t.session.Directory(), directory) + } + _, _ = fmt.Fprintf(output, " - install %s: %s\n", t.tracerLabel, shellquote.Join(install...)) } _, _ = fmt.Fprintf(output, " - run: %s\n", shellquote.Join(append([]string{command}, args...)...)) - _, _ = fmt.Fprintf(output, " - save a clickable report in %s\n", filepath.Join(directory, reportFilename)) - _, _ = fmt.Fprintf(output, " - save captured traffic in %s and test output in %s\n", filepath.Join(directory, "intake"), filepath.Join(directory, testOutputFilename)) _, _ = fmt.Fprintln(output) _, _ = fmt.Fprintln(output, "It will not change package.json, Gemfile, Python dependency files, or a lockfile in your project.") } @@ -224,7 +229,11 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { } _, _ = fmt.Fprintf(output, " %s: %s\n", displayName(t.framework.Name()), status) _, _ = fmt.Fprintf(output, " Tracer: %s\n", tracerLabel) - _, _ = fmt.Fprintf(output, "\nOpen report: %s\n", terminalLink(reportURL)) + reportLabel, err := filepath.Rel(t.repositoryRoot, reportPath) + if err != nil { + reportLabel = reportPath + } + _, _ = fmt.Fprintf(output, "\nOpen report: %s\n", terminalLink(reportURL, reportLabel)) if testErr != nil { return fmt.Errorf("%s failed after sending %d test event(s): %w", t.framework.Name(), findings.TestEventCount, testErr) diff --git a/internal/testdrive/testdrive_test.go b/internal/testdrive/testdrive_test.go index 65fd7faf..2c6c89b2 100644 --- a/internal/testdrive/testdrive_test.go +++ b/internal/testdrive/testdrive_test.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strconv" "strings" "testing" @@ -80,7 +81,10 @@ func writeJestManifest(t *testing.T, repositoryRoot string) { } func TestPrepareAndPreviewJest(t *testing.T) { - repositoryRoot := t.TempDir() + repositoryRoot := filepath.Join(t.TempDir(), "project space") + if err := os.Mkdir(repositoryRoot, 0755); err != nil { + t.Fatal(err) + } writeJestManifest(t, repositoryRoot) t.Chdir(repositoryRoot) @@ -95,13 +99,18 @@ func TestPrepareAndPreviewJest(t *testing.T) { "found JavaScript and Jest", "dd-trace@", "npx jest", - filepath.Join(repositoryRoot, ".testoptimization", "testdrive"), + filepath.Join(".testoptimization", "testdrive"), "will not change package.json", } { if !strings.Contains(output.String(), expected) { t.Errorf("Preview() output does not contain %q:\n%s", expected, output.String()) } } + for _, unwanted := range []string{repositoryRoot, "save a clickable report", "save captured traffic"} { + if strings.Contains(output.String(), unwanted) { + t.Errorf("Preview() contains %q:\n%s", unwanted, output.String()) + } + } } func TestPrepareRejectsUnsupportedRepository(t *testing.T) { @@ -210,7 +219,7 @@ func TestRunReportsCapturedTestsAndCoverage(t *testing.T) { "Jest: Passed", "Tracer: dd-trace@latest · isolated", "\x1b]8;;file://", - "report.html", + "\x1b\\" + filepath.Join(".testoptimization", "testdrive", filepath.Base(installer.sessionDirectory), "report.html") + "\x1b]8;;", } { if !strings.Contains(output.String(), expected) { t.Errorf("Run() output does not contain %q:\n%s", expected, output.String()) @@ -578,9 +587,13 @@ func TestPreviewChoosesTracerBeforeConfirmation(t *testing.T) { t.Fatal(err) } } else { - if !strings.Contains(preview, "npm install --prefix "+drive.session.Directory()+" --global=false --no-save --package-lock=false --no-audit --no-fund dd-trace@6.15.0") { + directory := filepath.Join(".testoptimization", "testdrive", drive.session.ID()) + if !strings.Contains(preview, "npm install --prefix "+directory+" --global=false --no-save --package-lock=false --no-audit --no-fund dd-trace@6.15.0") { t.Fatal(preview) } + if !slices.Contains(drive.installArgs, drive.session.Directory()) { + t.Fatalf("preview changed the actual installation path: %v", drive.installArgs) + } } }) } From c875aedcbb078b1accc1793547bd8f9af909fe59 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 16:13:32 +0200 Subject: [PATCH 08/11] Name discovered dependency files in testdrive previews --- internal/testdrive/testdrive.go | 26 ++++++++++++++- internal/testdrive/testdrive_test.go | 47 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index 608e9460..a3134178 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -135,7 +135,31 @@ func (t *Testdrive) Preview(output io.Writer) { _, _ = fmt.Fprintf(output, " - run: %s\n", shellquote.Join(append([]string{command}, args...)...)) _, _ = fmt.Fprintln(output) - _, _ = fmt.Fprintln(output, "It will not change package.json, Gemfile, Python dependency files, or a lockfile in your project.") + patterns := map[string][]string{ + "javascript": {"package.json", "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb"}, + "python": {"pyproject.toml", "setup.py", "setup.cfg", "requirements*.txt", "requirements*.in", "Pipfile", "Pipfile.lock", "poetry.lock", "uv.lock", "pdm.lock", "pylock.toml"}, + "ruby": {"Gemfile", "Gemfile.lock", "gems.rb", "gems.locked"}, + }[t.language] + entries, _ := os.ReadDir(t.repositoryRoot) + var files []string + for _, pattern := range patterns { + for _, entry := range entries { + matched, _ := filepath.Match(pattern, entry.Name()) + if !matched { + continue + } + if info, err := os.Stat(filepath.Join(t.repositoryRoot, entry.Name())); err == nil && !info.IsDir() { + files = append(files, entry.Name()) + } + } + } + if len(files) > 0 { + names := files[0] + if len(files) > 1 { + names = strings.Join(files[:len(files)-1], ", ") + " or " + files[len(files)-1] + } + _, _ = fmt.Fprintf(output, "It will not change %s.\n", names) + } } // Run prepares the tracer and executes the detected test suite. diff --git a/internal/testdrive/testdrive_test.go b/internal/testdrive/testdrive_test.go index 2c6c89b2..b6969fb5 100644 --- a/internal/testdrive/testdrive_test.go +++ b/internal/testdrive/testdrive_test.go @@ -113,6 +113,53 @@ func TestPrepareAndPreviewJest(t *testing.T) { } } +func TestPreviewNamesOnlyDiscoveredDependencyFiles(t *testing.T) { + for _, tc := range []struct { + name, language string + framework framework.Framework + files []string + want string + }{ + {"manifest only", "javascript", framework.NewJest(), []string{"package.json"}, "It will not change package.json."}, + {"npm", "javascript", framework.NewJest(), []string{"package.json", "package-lock.json"}, "It will not change package.json or package-lock.json."}, + {"pnpm with other languages", "javascript", framework.NewJest(), []string{"package.json", "pnpm-lock.yaml", "Gemfile", "pyproject.toml"}, "It will not change package.json or pnpm-lock.yaml."}, + {"yarn", "javascript", framework.NewJest(), []string{"package.json", "yarn.lock"}, "It will not change package.json or yarn.lock."}, + {"bun", "javascript", framework.NewJest(), []string{"package.json", "bun.lock"}, "It will not change package.json or bun.lock."}, + {"uv", "python", framework.NewPytest(), []string{"pyproject.toml", "uv.lock", "package.json"}, "It will not change pyproject.toml or uv.lock."}, + {"pip", "python", framework.NewPytest(), []string{"requirements.txt", "requirements-dev.txt"}, "It will not change requirements-dev.txt or requirements.txt."}, + {"poetry", "python", framework.NewPytest(), []string{"pyproject.toml", "poetry.lock"}, "It will not change pyproject.toml or poetry.lock."}, + {"ruby reused", "ruby", framework.NewRSpec(), []string{"Gemfile", "Gemfile.lock", "package.json"}, "It will not change Gemfile or Gemfile.lock."}, + {"multiple locks", "javascript", framework.NewJest(), []string{"package.json", "package-lock.json", "yarn.lock"}, "It will not change package.json, package-lock.json or yarn.lock."}, + {"no dependency files", "python", framework.NewPytest(), []string{"pytest.ini"}, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + root := filepath.Join(t.TempDir(), "project [space]") + if err := os.Mkdir(root, 0755); err != nil { + t.Fatal(err) + } + for _, name := range tc.files { + requireWriteFile(t, filepath.Join(root, name), "") + } + // A directory with a dependency filename is not a discovered file. + if err := os.Mkdir(filepath.Join(root, "bun.lockb"), 0755); err != nil { + t.Fatal(err) + } + drive := &Testdrive{repositoryRoot: root, language: tc.language, framework: tc.framework, session: planSession(root), projectTracer: "installed"} + var output bytes.Buffer + drive.Preview(&output) + var notice string + for line := range strings.SplitSeq(output.String(), "\n") { + if strings.HasPrefix(line, "It will not change") { + notice = line + } + } + if notice != tc.want { + t.Fatalf("dependency notice = %q, want %q", notice, tc.want) + } + }) + } +} + func TestPrepareRejectsUnsupportedRepository(t *testing.T) { t.Chdir(t.TempDir()) _, err := Prepare("latest") From 1615d584d28b99285b885a38da5f72cb5884c4ef Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 16:33:08 +0200 Subject: [PATCH 09/11] Show captured testdrive failure output without duplicate errors --- internal/testdrive/testdrive.go | 20 ++++++ internal/testdrive/testdrive_test.go | 104 ++++++++++++++++++--------- main.go | 2 +- main_test.go | 27 +++++++ 4 files changed, 118 insertions(+), 35 deletions(-) diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index a3134178..c4684735 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -210,6 +210,26 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { if err := os.WriteFile(testOutputPath, testOutput, 0644); err != nil { return fmt.Errorf("save test output: %w", err) } + if testErr != nil { + _, _ = fmt.Fprintf(output, "\n%s command output:\n", displayName(t.framework.Name())) + captured := strings.TrimSpace(string(testOutput)) + lines := strings.Split(captured, "\n") + switch { + case captured == "": + _, _ = fmt.Fprintln(output, "The command produced no output.") + case len(lines) > 80: + _, _ = fmt.Fprintln(output, strings.Join(lines[:40], "\n")) + _, _ = fmt.Fprintf(output, "\n... %d %s omitted; see the full test output below ...\n\n", len(lines)-80, plural(len(lines)-80, "line", "lines")) + _, _ = fmt.Fprintln(output, strings.Join(lines[len(lines)-40:], "\n")) + default: + _, _ = fmt.Fprintln(output, captured) + } + outputLabel, err := filepath.Rel(t.repositoryRoot, testOutputPath) + if err != nil { + outputLabel = testOutputPath + } + _, _ = fmt.Fprintf(output, "\nFull test output: %s\n", outputLabel) + } // Drain the intake before taking the snapshot used by the report. closeErr := server.Close() diff --git a/internal/testdrive/testdrive_test.go b/internal/testdrive/testdrive_test.go index b6969fb5..6f156d58 100644 --- a/internal/testdrive/testdrive_test.go +++ b/internal/testdrive/testdrive_test.go @@ -329,40 +329,76 @@ func TestRunReportsCapturedTestsAndCoverage(t *testing.T) { } func TestRunStillReportsEventsWhenJestFails(t *testing.T) { - repositoryRoot := t.TempDir() - writeJestManifest(t, repositoryRoot) - t.Chdir(repositoryRoot) - testdrive, err := Prepare("latest") - if err != nil { - t.Fatal(err) - } - - testdrive.platform = &fakeTracer{preloadPath: "/tmp/dd-trace/ci/init.js"} - testdrive.executor = &fakeTestdriveExecutor{output: []byte("FAIL one.test.js\n"), err: errors.New("exit status 1")} - testdrive.startIntake = func(string) (localIntake, error) { - return &fakeIntake{ - url: "http://127.0.0.1:1234", - findings: intake.Facts{ - TestCount: 1, - TestEventCount: 1, - FailedTests: []intake.Test{{ - Name: "fails", Suite: "one.test.js", Status: "fail", - Attempts: []intake.TestRun{{Status: "fail", Duration: time.Millisecond}}, - }}, - }, - }, nil - } - - var output bytes.Buffer - err = testdrive.Run(t.Context(), &output) - if err == nil || !strings.Contains(err.Error(), "jest failed after sending 1 test event") { - t.Fatalf("Run() error = %v", err) - } - if !strings.Contains(output.String(), "Test events received.") { - t.Fatalf("Run() did not report working instrumentation:\n%s", output.String()) - } - if !strings.Contains(output.String(), "Failed tests (1):") || !strings.Contains(output.String(), "one.test.js › fails · Fail · 1ms") || !strings.Contains(output.String(), "Jest: Failed") || !strings.Contains(output.String(), "file://") { - t.Fatalf("Run() did not report the failure and report link:\n%s", output.String()) + for _, tc := range []struct { + name string + output string + events int + want string + }{ + {name: "assertion failure", output: "FAIL one.test.js\nExpected: true\nReceived: false\n at one.test.js:4:18\n", events: 1, want: "Expected: true\nReceived: false\n at one.test.js:4:18"}, + {name: "suite setup failure without failed events", output: "Cannot find module './missing' from 'setup.js'\n", events: 1363, want: "Cannot find module './missing'"}, + {name: "no events", output: "SyntaxError: unexpected token in jest.config.js\n", want: "SyntaxError: unexpected token"}, + {name: "empty output", events: 1, want: "The command produced no output."}, + {name: "80 lines", output: strings.Repeat("log line\n", 79) + "final failure", events: 1, want: "final failure"}, + {name: "long output", output: "initial failure\n" + strings.Repeat("log line\n", 79) + "final failure\n", events: 1, want: "... 1 line omitted; see the full test output below ..."}, + } { + t.Run(tc.name, func(t *testing.T) { + repositoryRoot := filepath.Join(t.TempDir(), "project space") + if err := os.Mkdir(repositoryRoot, 0755); err != nil { + t.Fatal(err) + } + writeJestManifest(t, repositoryRoot) + t.Chdir(repositoryRoot) + testdrive, err := Prepare("latest") + if err != nil { + t.Fatal(err) + } + installer := &fakeTracer{preloadPath: "/tmp/dd-trace/ci/init.js"} + testdrive.platform = installer + failure := errors.New("exit status 1") + testdrive.executor = &fakeTestdriveExecutor{output: []byte(tc.output), err: failure} + findings := intake.Facts{TestCount: tc.events, TestEventCount: tc.events} + if tc.name == "assertion failure" { + findings.FailedTests = []intake.Test{{Name: "fails", Suite: "one.test.js", Status: "fail", Attempts: []intake.TestRun{{Status: "fail", Duration: time.Millisecond}}}} + } + testdrive.startIntake = func(string) (localIntake, error) { + return &fakeIntake{url: "http://127.0.0.1:1234", findings: findings}, nil + } + var output bytes.Buffer + err = testdrive.Run(t.Context(), &output) + if !errors.Is(err, failure) { + t.Fatalf("Run() error = %v", err) + } + label := filepath.Join(".testoptimization", "testdrive", filepath.Base(installer.sessionDirectory), testOutputFilename) + for _, expected := range []string{"Jest command output:", tc.want, "Full test output: " + label, "Open report:"} { + if !strings.Contains(output.String(), expected) { + t.Errorf("missing %q in output:\n%s", expected, output.String()) + } + } + if tc.events > 0 && !strings.Contains(output.String(), "Test events received.") { + t.Fatal(output.String()) + } + if tc.name == "assertion failure" && !strings.Contains(output.String(), "one.test.js › fails · Fail · 1ms") { + t.Fatal(output.String()) + } + if tc.name == "long output" { + if !strings.Contains(output.String(), "initial failure") || !strings.Contains(output.String(), "final failure") || strings.Count(output.String(), "log line") != 78 { + t.Fatal(output.String()) + } + } else if strings.Contains(output.String(), "lines omitted") { + t.Fatal(output.String()) + } + contents, err := os.ReadFile(label) + if err != nil { + t.Fatal(err) + } + if string(contents) != tc.output { + t.Fatalf("saved output changed: %q", contents) + } + if strings.Contains(output.String(), "Full test output: "+repositoryRoot) { + t.Fatal("output path is absolute") + } + }) } } diff --git a/main.go b/main.go index 8c22910e..0d435b77 100644 --- a/main.go +++ b/main.go @@ -36,7 +36,7 @@ func run(execute func() error) int { _ = os.Setenv("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", "0") if err := execute(); err != nil { - slog.Error("FAILURE", "error", err) + // Cobra has already printed the error. var exitErr *exec.ExitError if errors.As(err, &exitErr) { if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() { diff --git a/main_test.go b/main_test.go index 8e98b85b..4e0cfd47 100644 --- a/main_test.go +++ b/main_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "runtime" + "strings" "testing" ) @@ -70,3 +71,29 @@ func TestRunPreservesProcessSignal(t *testing.T) { t.Fatalf("run() exit code = %d, want 143", code) } } + +func TestCLIPrintsErrorsOnce(t *testing.T) { + if scenario := os.Getenv("DDTEST_ERROR_HELPER"); scenario != "" { + os.Args = []string{"ddtest", "testdrive", scenario} + os.Exit(run(executeCommand)) + } + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, scenario := range []string{"--unknown", "--yes"} { + t.Run(scenario, func(t *testing.T) { + command := exec.Command(executable, "-test.run=^TestCLIPrintsErrorsOnce$") + command.Dir = t.TempDir() + command.Env = append(os.Environ(), "DDTEST_ERROR_HELPER="+scenario) + output, err := command.CombinedOutput() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("error = %v, output = %s", err, output) + } + if strings.Count(string(output), "Error:") != 1 || strings.Contains(string(output), "FAILURE") { + t.Fatalf("expected one error, got:\n%s", output) + } + }) + } +} From 03a55404bce0caaa8c02f20d0fa6ebd58a7333bd Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 16:41:11 +0200 Subject: [PATCH 10/11] Label the Datadog library in testdrive run details --- internal/testdrive/report.go | 2 +- internal/testdrive/report_test.go | 2 +- internal/testdrive/testdrive.go | 2 +- internal/testdrive/testdrive_test.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index 7634ac3a..1887cdde 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -56,7 +56,7 @@ func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime .. {"Test events", fmt.Sprint(findings.TestEventCount)}, {"Tests with coverage", coverage}, {info.Framework, status}, - {"Tracer", info.Tracer}, + {"Datadog library", info.Tracer}, }} if findings.TestEventCount == 0 { model.Headline = "No test events received." diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index 527c8e27..b7e71ab0 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -113,7 +113,7 @@ func TestReportRuntimeFacts(t *testing.T) { for _, fact := range model.Facts { facts[fact.Label] = fact.Value } - if facts["Jest"] != tc.wantStatus || facts["Tracer"] != tc.tracer { + if facts["Jest"] != tc.wantStatus || facts["Datadog library"] != tc.tracer { t.Fatalf("incorrect runtime facts: %v", facts) } if tc.events == 0 && (model.Headline != "No test events received." || strings.Contains(model.Summary, "No findings.")) { diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index c4684735..ac13eb82 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -272,7 +272,7 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { status = "No test results received" } _, _ = fmt.Fprintf(output, " %s: %s\n", displayName(t.framework.Name()), status) - _, _ = fmt.Fprintf(output, " Tracer: %s\n", tracerLabel) + _, _ = fmt.Fprintf(output, " Datadog library: %s\n", tracerLabel) reportLabel, err := filepath.Rel(t.repositoryRoot, reportPath) if err != nil { reportLabel = reportPath diff --git a/internal/testdrive/testdrive_test.go b/internal/testdrive/testdrive_test.go index 6f156d58..d95fed47 100644 --- a/internal/testdrive/testdrive_test.go +++ b/internal/testdrive/testdrive_test.go @@ -264,7 +264,7 @@ func TestRunReportsCapturedTestsAndCoverage(t *testing.T) { "Test events: 2", "Tests with coverage: 2 / 2", "Jest: Passed", - "Tracer: dd-trace@latest · isolated", + "Datadog library: dd-trace@latest · isolated", "\x1b]8;;file://", "\x1b\\" + filepath.Join(".testoptimization", "testdrive", filepath.Base(installer.sessionDirectory), "report.html") + "\x1b]8;;", } { @@ -619,7 +619,7 @@ func TestRunReportsProjectTracer(t *testing.T) { if installer.options.Version != "git:ignored-for-existing-tracer" || installer.options.Command != drive.command { t.Fatal(installer.options) } - if !strings.Contains(output.String(), "Tracer: dd-trace · reused") { + if !strings.Contains(output.String(), "Datadog library: dd-trace · reused") { t.Fatal(output.String()) } if !strings.Contains(executor.env["NODE_OPTIONS"], installer.preloadPath) { From 93dced666e6a3d7903ec9958f8df422d702c98c1 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 16:47:29 +0200 Subject: [PATCH 11/11] Require five-second and five-times-median slow test thresholds --- internal/testdrive/intake/findings.go | 5 ++- internal/testdrive/intake/findings_test.go | 40 +++++++++++++++++++--- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go index a95fa68b..1a5a7252 100644 --- a/internal/testdrive/intake/findings.go +++ b/internal/testdrive/intake/findings.go @@ -19,7 +19,7 @@ import ( ) const ( - minimumSlowDuration = 100 * time.Millisecond + minimumSlowDuration = 5 * time.Second minimumBroadCoverageFiles = 5 ) @@ -215,11 +215,10 @@ func slowTests(tests []Test) ([]Test, time.Duration) { return nil, medianTestDuration(tests) } median := medianTestDuration(tests) - threshold := max(minimumSlowDuration, median*2) slow := make([]Test, 0) for _, test := range tests { - if test.Duration >= threshold && test.Duration > median { + if test.Duration > minimumSlowDuration && test.Duration >= median*5 { slow = append(slow, test) } } diff --git a/internal/testdrive/intake/findings_test.go b/internal/testdrive/intake/findings_test.go index 80998472..45177c60 100644 --- a/internal/testdrive/intake/findings_test.go +++ b/internal/testdrive/intake/findings_test.go @@ -21,7 +21,7 @@ func TestAnalyzeTestsFindsFailuresRetriesAndSlowTests(t *testing.T) { {name: "fast", suite: "one.test.js", status: "pass", duration: 10 * time.Millisecond}, {name: "flaky", suite: "one.test.js", status: "fail", duration: 20 * time.Millisecond}, {name: "flaky", suite: "one.test.js", status: "pass", duration: 30 * time.Millisecond, isRetry: true}, - {name: "slow", suite: "two.test.js", status: "pass", duration: 400 * time.Millisecond}, + {name: "slow", suite: "two.test.js", status: "pass", duration: 6 * time.Second}, {name: "broken", suite: "two.test.js", status: "fail", duration: 15 * time.Millisecond}, } @@ -40,12 +40,44 @@ func TestAnalyzeTestsFindsFailuresRetriesAndSlowTests(t *testing.T) { }, }}, flaky) require.Equal(t, []Test{{ - Name: "slow", Suite: "two.test.js", Status: "pass", Duration: 400 * time.Millisecond, - Attempts: []TestRun{{Status: "pass", Duration: 400 * time.Millisecond}}, + Name: "slow", Suite: "two.test.js", Status: "pass", Duration: 6 * time.Second, + Attempts: []TestRun{{Status: "pass", Duration: 6 * time.Second}}, }}, slow) require.Equal(t, 17500*time.Microsecond, median) } +func TestAnalyzeTestsSlowThresholds(t *testing.T) { + for _, tc := range []struct { + name string + median time.Duration + duration time.Duration + wantSlow bool + }{ + {name: "fast outlier", median: 10 * time.Millisecond, duration: 400 * time.Millisecond}, + {name: "exactly five seconds", median: time.Second, duration: 5 * time.Second}, + {name: "above five seconds", median: time.Second, duration: 5*time.Second + time.Nanosecond, wantSlow: true}, + {name: "below five times median", median: 2 * time.Second, duration: 10*time.Second - time.Nanosecond}, + {name: "exactly five times median", median: 2 * time.Second, duration: 10 * time.Second, wantSlow: true}, + {name: "above five times median", median: 2 * time.Second, duration: 11 * time.Second, wantSlow: true}, + {name: "uniformly slow tests", median: 6 * time.Second, duration: 6 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, _, slow, median := analyzeTests([]testReference{ + {name: "baseline one", status: "pass", duration: tc.median}, + {name: "baseline two", status: "pass", duration: tc.median}, + {name: "candidate", status: "pass", duration: tc.duration}, + }, nil, "") + require.Equal(t, tc.median, median) + if tc.wantSlow { + require.Len(t, slow, 1) + require.Equal(t, "candidate", slow[0].Name) + } else { + require.Empty(t, slow) + } + }) + } +} + func TestAnalyzeTestsUsesFinalStatusAndMarksMixedOutcomesFlaky(t *testing.T) { tests := []testReference{ {name: "flaky", suite: "one.test.js", status: "pass", finalStatus: "pass", duration: 10 * time.Millisecond}, @@ -250,7 +282,7 @@ func TestFindingsPreservesCoverageInEveryCategory(t *testing.T) { } duration := time.Millisecond if name == "" { - duration = time.Second + duration = 6 * time.Second } events = append(events, map[string]any{"type": "test", "content": map[string]any{ "test_session_id": 1, "test_suite_id": 2, "span_id": i + 1, "duration": int64(duration),