From c04419087bf8303086abef9917dc7329b3e8055e Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:14:18 +0200 Subject: [PATCH 01/35] Support Mocha, Vitest, Playwright and Cucumber testdrives --- internal/framework/testdrive_test.go | 49 +++++++ .../testdrive/frameworks_integration_test.go | 123 ++++++++++++++++++ internal/testdrive/multiframework_test.go | 88 +++++++++++++ 3 files changed, 260 insertions(+) create mode 100644 internal/framework/testdrive_test.go create mode 100644 internal/testdrive/frameworks_integration_test.go create mode 100644 internal/testdrive/multiframework_test.go diff --git a/internal/framework/testdrive_test.go b/internal/framework/testdrive_test.go new file mode 100644 index 00000000..0f7a620e --- /dev/null +++ b/internal/framework/testdrive_test.go @@ -0,0 +1,49 @@ +package framework + +import ( + "os" + "testing" + + "github.com/DataDog/ddtest/internal/settings" + "github.com/stretchr/testify/require" +) + +func TestTestdriveUsesFrameworkCommand(t *testing.T) { + old := settings.Get().Command + settings.Get().Command = "" + t.Cleanup(func() { settings.Get().Command = old }) + for _, tc := range []struct { + runner Framework + args []string + }{ + {NewJest(), []string{"jest"}}, + {NewMocha(), []string{"mocha"}}, + {NewVitest(), []string{"vitest", "run"}}, + {NewPlaywright(), []string{"playwright", "test"}}, + {NewCucumber(), []string{"cucumber-js"}}, + } { + t.Run(tc.runner.Name(), func(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + // Neither a custom script nor a package manager lockfile overrides execution. + require.NoError(t, os.WriteFile("package.json", []byte(`{"scripts":{"test":"jest && echo side-effect","unit":"vitest --config custom.ts"}}`), 0644)) + require.NoError(t, os.WriteFile("yarn.lock", nil, 0644)) + command, args, err := TestdriveCommand(root, tc.runner) + require.NoError(t, err) + require.Equal(t, "npx", command) + require.Equal(t, tc.args, args) + }) + } +} + +func TestTestdrivePreservesExplicitCommandArguments(t *testing.T) { + t.Cleanup(func() { settings.Get().Command = "" }) + settings.Get().Command = `npm run smoke -- --config "config with spaces.js"` + command, args, err := TestdriveCommand(t.TempDir(), NewMocha()) + require.NoError(t, err) + require.Equal(t, "npm", command) + require.Equal(t, []string{"run", "smoke", "--", "--config", "config with spaces.js"}, args) + settings.Get().Command = `npm "` + _, _, err = TestdriveCommand(t.TempDir(), NewMocha()) + require.ErrorContains(t, err, "parse testdrive") +} diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go new file mode 100644 index 00000000..4b847fb5 --- /dev/null +++ b/internal/testdrive/frameworks_integration_test.go @@ -0,0 +1,123 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestPublicFrameworkTestdrives exercises the shipped CLI, real tracers, and +// real test runners. Browser downloads are explicit test setup, never testdrive +// side effects. Run with DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1. +func TestPublicFrameworkTestdrives(t *testing.T) { + if os.Getenv("DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST") == "" { + t.Skip("set DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1") + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) + defer cancel() + binary := filepath.Join(t.TempDir(), "ddtest") + integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + fixtures := []struct { + name, manifest, command string + files map[string]string + }{ + {"jest", `{"scripts":{"test":"jest"},"devDependencies":{"jest":"30.5.1"}}`, "npm test", map[string]string{"one.test.js": `test('adds', () => expect(1+1).toBe(2));`}}, + {"mocha", `{"scripts":{"test":"mocha"},"devDependencies":{"mocha":"11.7.5"}}`, "npm test", map[string]string{"test/one.js": `const assert = require('node:assert'); it('adds', () => assert.equal(1+1,2));`}}, + {"vitest", `{"type":"module","scripts":{"test":"vitest run"},"devDependencies":{"vitest":"3.2.4"}}`, "npm test", map[string]string{"one.test.js": `import {test,expect} from 'vitest'; test('adds', () => expect(1+1).toBe(2));`}}, + {"playwright", `{"scripts":{"test":"playwright test"},"devDependencies":{"@playwright/test":"1.55.1"}}`, "npm test", map[string]string{"one.spec.js": `const {test,expect} = require('@playwright/test'); test('adds', () => expect(1+1).toBe(2));`}}, + {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Background:\n Given addition works\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, + } + for _, fixture := range fixtures { + t.Run(fixture.name, func(t *testing.T) { + name := "project space" + if fixture.name == "rspec" || fixture.name == "minitest" { + name = "project" + } // Ruby native extension builds do not support spaces. + root := filepath.Join(t.TempDir(), name) + require.NoError(t, os.MkdirAll(root, 0755)) + integrationCommand(t, ctx, root, nil, "git", "init", "-q") + for name, contents := range fixture.files { + integrationFile(t, root, name, contents) + } + if fixture.manifest != "" { + integrationFile(t, root, "package.json", fixture.manifest) + } + integrationFile(t, root, ".github/workflows/test.yml", "name: tests\non: push\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: "+fixture.command+"\n") + env := []string{} + if fixture.manifest != "" { + integrationCommand(t, ctx, root, env, "npm", "install", "--no-audit", "--no-fund") + } + if fixture.name == "pytest" { + venv := filepath.Join(t.TempDir(), "venv") + integrationCommand(t, ctx, root, nil, "python3", "-m", "venv", venv) + env = append(env, "PATH="+filepath.Join(venv, "bin")+string(os.PathListSeparator)+os.Getenv("PATH")) + integrationCommand(t, ctx, root, env, filepath.Join(venv, "bin", "python"), "-m", "pip", "install", "pytest==8.4.2") + } + if fixture.name == "rspec" || fixture.name == "minitest" { + integrationCommand(t, ctx, root, env, "bundle", "lock") + } + before := map[string]string{} + for _, name := range []string{"package.json", "package-lock.json", "Gemfile", "Gemfile.lock", "requirements.txt", "cypress.config.js"} { + contents, err := os.ReadFile(filepath.Join(root, name)) + if err == nil { + before[name] = string(contents) + } + } + output := integrationCommand(t, ctx, root, env, binary, "testdrive", "--yes") + require.Contains(t, output, "Test events received.") + require.Contains(t, output, "Open report:") + reports, err := filepath.Glob(filepath.Join(root, ".testoptimization", "testdrive", "*", "report.html")) + require.NoError(t, err) + require.Len(t, reports, 1) + contents, err := os.ReadFile(reports[0]) + require.NoError(t, err) + require.Contains(t, string(contents), "Test events received.") + traffic, err := filepath.Glob(filepath.Join(filepath.Dir(reports[0]), "intake", "*citestcycle.json")) + require.NoError(t, err) + require.NotEmpty(t, traffic) + for name, contents := range before { + after, err := os.ReadFile(filepath.Join(root, name)) + require.NoError(t, err) + require.Equal(t, contents, string(after), name) + } + if _, existed := before["Gemfile.lock"]; !existed { + _, err := os.Stat(filepath.Join(root, "Gemfile.lock")) + require.True(t, os.IsNotExist(err), "project lockfile must not be created") + } + if fixture.name == "cypress" { + hook, err := os.ReadFile(filepath.Join(root, "original-hook.txt")) + require.NoError(t, err) + require.Equal(t, "ran", string(hook)) + } + t.Log(strings.TrimSpace(output)) + }) + } +} + +func integrationFile(t *testing.T, root, name, contents string) { + t.Helper() + path := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, []byte(contents), 0644)) +} + +func integrationCommand(t *testing.T, ctx context.Context, directory string, env []string, name string, args ...string) string { + t.Helper() + command := exec.CommandContext(ctx, name, args...) + command.Dir = directory + command.Env = append(os.Environ(), env...) + output, err := command.CombinedOutput() + require.NoError(t, err, "%s %v:\n%s", name, args, output) + return string(output) +} diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go new file mode 100644 index 00000000..03c4033e --- /dev/null +++ b/internal/testdrive/multiframework_test.go @@ -0,0 +1,88 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "github.com/DataDog/ddtest/internal/settings" + "os" + "path/filepath" + "testing" + + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/stretchr/testify/require" +) + +func TestPrepareAllSupportedFrameworks(t *testing.T) { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + switch name { + case "pytest": + requireWriteFile(t, filepath.Join(root, "pyproject.toml"), "[tool.pytest.ini_options]\n") + case "rspec", "minitest": + requireWriteFile(t, filepath.Join(root, "Gemfile"), "gem '"+name+"'\n") + default: + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"`+name+`"}}`) + } + t.Chdir(root) + run, err := Prepare() + require.NoError(t, err) + var preview bytes.Buffer + run.Preview(&preview) + require.Contains(t, preview.String(), displayName(name)) + _, err = os.Stat(filepath.Join(root, ".testoptimization")) + require.True(t, os.IsNotExist(err), "preview must be read-only") + if name == "cypress" { + return + } // Browser wrapper has its own real-run test. + run.tracer = &fakeTracer{preloadPath: filepath.Join(root, "isolated")} + executor := &fakeTestdriveExecutor{} + run.executor = executor + run.startIntake = func(string) (localIntake, error) { + return &fakeIntake{url: "http://127.0.0.1:1234", findings: intake.Findings{TestEventCount: 1, TestCount: 1}}, nil + } + var output bytes.Buffer + require.NoError(t, run.Run(t.Context(), &output)) + require.Contains(t, output.String(), displayName(name)+": Passed") + require.Contains(t, output.String(), "Tests with coverage: 0 / 1") + require.Equal(t, "ddtest-testdrive", executor.env["DD_API_KEY"]) + if name == "cucumber" { + require.Equal(t, "false", executor.env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"]) + } + switch run.language { + case "javascript": + require.Contains(t, executor.env["NODE_OPTIONS"], "--import") + case "python": + require.NotEmpty(t, executor.env["PYTHONPATH"]) + require.Contains(t, executor.env["PYTEST_ADDOPTS"], "--ddtrace") + require.NotContains(t, executor.env, "NODE_OPTIONS") + case "ruby": + require.NotEmpty(t, executor.env["BUNDLE_GEMFILE"]) + require.Contains(t, executor.env["RUBYOPT"], "datadog/ci/auto_instrument") + require.NotContains(t, executor.env, "NODE_OPTIONS") + } + }) + } +} + +func TestPrepareRequiresSelectionForMultipleFrameworks(t *testing.T) { + previous := settings.Get().Framework + t.Cleanup(func() { settings.Get().Framework = previous }) + settings.Get().Framework = "" + root := t.TempDir() + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"vitest run","e2e":"playwright test"}}`) + t.Chdir(root) + _, err := Prepare() + require.ErrorContains(t, err, "--framework") + settings.Get().Framework = "vitest" + run, err := Prepare() + require.NoError(t, err) + require.Equal(t, "vitest", run.framework.Name()) + settings.Get().Framework = "unsupported" + _, err = Prepare() + require.ErrorContains(t, err, "unsupported framework") +} From 278b13cdafc02e3529627f2b47b2a186df9a594a Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 23 Sep 2026 17:02:32 +0200 Subject: [PATCH 02/35] Use renamed intake fact types --- internal/testdrive/multiframework_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 03c4033e..50b0e639 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -43,7 +43,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { executor := &fakeTestdriveExecutor{} run.executor = executor run.startIntake = func(string) (localIntake, error) { - return &fakeIntake{url: "http://127.0.0.1:1234", findings: intake.Findings{TestEventCount: 1, TestCount: 1}}, nil + return &fakeIntake{url: "http://127.0.0.1:1234", findings: intake.Facts{TestEventCount: 1, TestCount: 1}}, nil } var output bytes.Buffer require.NoError(t, run.Run(t.Context(), &output)) From 064bf59092b40fbfcc127da3dba1f392e29b4d1b Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 09:12:00 +0200 Subject: [PATCH 03/35] Gate ESM preload by Node support --- internal/testdrive/multiframework_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 50b0e639..d3942ead 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -39,6 +39,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { if name == "cypress" { return } // Browser wrapper has its own real-run test. + run.nodeVersion = func() string { return "v20.0.0" } run.tracer = &fakeTracer{preloadPath: filepath.Join(root, "isolated")} executor := &fakeTestdriveExecutor{} run.executor = executor @@ -69,6 +70,19 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { } } +func TestSupportsNodeImport(t *testing.T) { + for version, want := range map[string]bool{ + "v18.17.1": false, + "v18.18.0": true, + "v20.0.0": true, + "invalid": false, + } { + if got := supportsNodeImport(version); got != want { + t.Errorf("supportsNodeImport(%q) = %v, want %v", version, got, want) + } + } +} + func TestPrepareRequiresSelectionForMultipleFrameworks(t *testing.T) { previous := settings.Get().Framework t.Cleanup(func() { settings.Get().Framework = previous }) From eb1155d02f56fc9b3c5cfcc4e02a04292c429e93 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 13:32:32 +0200 Subject: [PATCH 04/35] Keep testdrive fixtures aligned with latest tracer selection --- internal/testdrive/multiframework_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index d3942ead..8dee2d1e 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -29,7 +29,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"`+name+`"}}`) } t.Chdir(root) - run, err := Prepare() + run, err := Prepare("latest") require.NoError(t, err) var preview bytes.Buffer run.Preview(&preview) @@ -90,13 +90,13 @@ func TestPrepareRequiresSelectionForMultipleFrameworks(t *testing.T) { root := t.TempDir() requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"vitest run","e2e":"playwright test"}}`) t.Chdir(root) - _, err := Prepare() + _, err := Prepare("latest") require.ErrorContains(t, err, "--framework") settings.Get().Framework = "vitest" - run, err := Prepare() + run, err := Prepare("latest") require.NoError(t, err) require.Equal(t, "vitest", run.framework.Name()) settings.Get().Framework = "unsupported" - _, err = Prepare() + _, err = Prepare("latest") require.ErrorContains(t, err, "unsupported framework") } From 327c3badf8e7dcbcdb2b3e8a7ae880fa83bb9885 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 15:30:39 +0200 Subject: [PATCH 05/35] Use Platform-owned tracer preparation in 142 --- internal/testdrive/multiframework_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 8dee2d1e..b9564041 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -40,7 +40,11 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { return } // Browser wrapper has its own real-run test. run.nodeVersion = func() string { return "v20.0.0" } - run.tracer = &fakeTracer{preloadPath: filepath.Join(root, "isolated")} + installer := &fakeTracer{preloadPath: filepath.Join(root, "isolated")} + if run.language == "ruby" { + installer.env = map[string]string{"BUNDLE_GEMFILE": filepath.Join(root, "isolated", "Gemfile")} + } + run.platform = installer executor := &fakeTestdriveExecutor{} run.executor = executor run.startIntake = func(string) (localIntake, error) { From 5076e4f1d1198c3829538f101551cfdcd31a71a0 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 17:01:14 +0200 Subject: [PATCH 06/35] Use the project Ruby bundle for testdrive --- .../command_test.go} | 24 +++++------ .../testdrive/frameworks_integration_test.go | 12 +++++- internal/testdrive/javascript.go | 43 +++++++++++++++++++ internal/testdrive/multiframework_test.go | 7 ++- internal/testdrive/testdrive.go | 10 +++-- 5 files changed, 72 insertions(+), 24 deletions(-) rename internal/{framework/testdrive_test.go => testdrive/command_test.go} (66%) diff --git a/internal/framework/testdrive_test.go b/internal/testdrive/command_test.go similarity index 66% rename from internal/framework/testdrive_test.go rename to internal/testdrive/command_test.go index 0f7a620e..7a82ed3e 100644 --- a/internal/framework/testdrive_test.go +++ b/internal/testdrive/command_test.go @@ -1,9 +1,10 @@ -package framework +package testdrive import ( "os" "testing" + "github.com/DataDog/ddtest/internal/framework" "github.com/DataDog/ddtest/internal/settings" "github.com/stretchr/testify/require" ) @@ -13,14 +14,14 @@ func TestTestdriveUsesFrameworkCommand(t *testing.T) { settings.Get().Command = "" t.Cleanup(func() { settings.Get().Command = old }) for _, tc := range []struct { - runner Framework + runner framework.Framework args []string }{ - {NewJest(), []string{"jest"}}, - {NewMocha(), []string{"mocha"}}, - {NewVitest(), []string{"vitest", "run"}}, - {NewPlaywright(), []string{"playwright", "test"}}, - {NewCucumber(), []string{"cucumber-js"}}, + {framework.NewJest(), []string{"jest"}}, + {framework.NewMocha(), []string{"mocha"}}, + {framework.NewVitest(), []string{"vitest", "run"}}, + {framework.NewPlaywright(), []string{"playwright", "test"}}, + {framework.NewCucumber(), []string{"cucumber-js"}}, } { t.Run(tc.runner.Name(), func(t *testing.T) { root := t.TempDir() @@ -28,8 +29,7 @@ func TestTestdriveUsesFrameworkCommand(t *testing.T) { // Neither a custom script nor a package manager lockfile overrides execution. require.NoError(t, os.WriteFile("package.json", []byte(`{"scripts":{"test":"jest && echo side-effect","unit":"vitest --config custom.ts"}}`), 0644)) require.NoError(t, os.WriteFile("yarn.lock", nil, 0644)) - command, args, err := TestdriveCommand(root, tc.runner) - require.NoError(t, err) + command, args := tc.runner.Command() require.Equal(t, "npx", command) require.Equal(t, tc.args, args) }) @@ -39,11 +39,7 @@ func TestTestdriveUsesFrameworkCommand(t *testing.T) { func TestTestdrivePreservesExplicitCommandArguments(t *testing.T) { t.Cleanup(func() { settings.Get().Command = "" }) settings.Get().Command = `npm run smoke -- --config "config with spaces.js"` - command, args, err := TestdriveCommand(t.TempDir(), NewMocha()) - require.NoError(t, err) + command, args := framework.NewMocha().Command() require.Equal(t, "npm", command) require.Equal(t, []string{"run", "smoke", "--", "--config", "config with spaces.js"}, args) - settings.Get().Command = `npm "` - _, _, err = TestdriveCommand(t.TempDir(), NewMocha()) - require.ErrorContains(t, err, "parse testdrive") } diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index 4b847fb5..5767f33a 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -43,7 +43,7 @@ func TestPublicFrameworkTestdrives(t *testing.T) { name := "project space" if fixture.name == "rspec" || fixture.name == "minitest" { name = "project" - } // Ruby native extension builds do not support spaces. + } // Keep the recorded Ruby fixture path; build failures retain Bundler diagnostics. root := filepath.Join(t.TempDir(), name) require.NoError(t, os.MkdirAll(root, 0755)) integrationCommand(t, ctx, root, nil, "git", "init", "-q") @@ -87,11 +87,19 @@ func TestPublicFrameworkTestdrives(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, traffic) for name, contents := range before { + if (fixture.name == "rspec" || fixture.name == "minitest") && (name == "Gemfile" || name == "Gemfile.lock") { + continue // bundle add updates Ruby dependency files. + } after, err := os.ReadFile(filepath.Join(root, name)) require.NoError(t, err) require.Equal(t, contents, string(after), name) } - if _, existed := before["Gemfile.lock"]; !existed { + if fixture.name == "rspec" || fixture.name == "minitest" { + gemfile, err := os.ReadFile(filepath.Join(root, "Gemfile")) + require.NoError(t, err) + require.Contains(t, string(gemfile), "datadog-ci") + require.FileExists(t, filepath.Join(root, "Gemfile.lock")) + } else if _, existed := before["Gemfile.lock"]; !existed { _, err := os.Stat(filepath.Join(root, "Gemfile.lock")) require.True(t, os.IsNotExist(err), "project lockfile must not be created") } diff --git a/internal/testdrive/javascript.go b/internal/testdrive/javascript.go index ccc2b159..b220b844 100644 --- a/internal/testdrive/javascript.go +++ b/internal/testdrive/javascript.go @@ -3,6 +3,7 @@ package testdrive import ( "encoding/json" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -66,3 +67,45 @@ func javascriptTracerVersion(preload string) string { } return pkg.Version } + +func currentNodeVersion() string { + output, err := exec.Command("node", "--version").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(output)) +} + +func supportsNodeImport(version string) bool { + version = strings.TrimPrefix(strings.TrimSpace(version), "v") + parts := strings.Split(version, ".") + if len(parts) < 2 { + return false + } + major, majorErr := strconv.Atoi(parts[0]) + minor, minorErr := strconv.Atoi(parts[1]) + if majorErr != nil || minorErr != nil { + return false + } + return major > 18 || major == 18 && minor >= 18 +} + +func (t *Testdrive) javascriptEnvironment(path string) map[string]string { + env := javascriptEnvironment(path) + // ESM instrumentation is needed by Vitest and by ESM test/config files. + version := "" + if t.nodeVersion != nil { + version = t.nodeVersion() + } + if supportsNodeImport(version) { + register := absoluteFileURL(filepath.Join(filepath.Dir(filepath.Dir(path)), "register.js")) + env["NODE_OPTIONS"] += " --import " + strconv.Quote(register) + } + // dd-trace 6.15.0 impacted-test detection dereferences scenario.id on + // Background/Rule nodes. Basic Cucumber reporting works with it off. + if t.framework.Name() == "cucumber" { + env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"] = "false" + } + + return env +} diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index b9564041..4c1e70cb 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -41,9 +41,6 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { } // Browser wrapper has its own real-run test. run.nodeVersion = func() string { return "v20.0.0" } installer := &fakeTracer{preloadPath: filepath.Join(root, "isolated")} - if run.language == "ruby" { - installer.env = map[string]string{"BUNDLE_GEMFILE": filepath.Join(root, "isolated", "Gemfile")} - } run.platform = installer executor := &fakeTestdriveExecutor{} run.executor = executor @@ -66,7 +63,9 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { require.Contains(t, executor.env["PYTEST_ADDOPTS"], "--ddtrace") require.NotContains(t, executor.env, "NODE_OPTIONS") case "ruby": - require.NotEmpty(t, executor.env["BUNDLE_GEMFILE"]) + require.NotContains(t, executor.env, "BUNDLE_GEMFILE") + require.Contains(t, preview.String(), "Bundler updates the project Gemfile and lockfile") + require.Contains(t, output.String(), "datadog-ci · installed in project") require.Contains(t, executor.env["RUBYOPT"], "datadog/ci/auto_instrument") require.NotContains(t, executor.env, "NODE_OPTIONS") } diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index e97244f9..5561571c 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -54,6 +54,7 @@ type Testdrive struct { installArgs []string executor commandExecutor startIntake func(string) (localIntake, error) + nodeVersion func() string } // Prepare detects the repository and probes the project tracer without writing files. @@ -78,7 +79,7 @@ func Prepare(version string) (*Testdrive, error) { } language := detectedPlatform.Name() switch runner.Name() { - case "jest": + case "jest", "mocha", "vitest", "playwright", "cucumber": default: return nil, fmt.Errorf("testdrive does not yet support %s", runner.Name()) } @@ -103,7 +104,7 @@ func Prepare(version string) (*Testdrive, error) { } return &Testdrive{projectTracer: projectTracer, session: session, installCommand: installCommand, installArgs: installArgs, repositoryRoot: repositoryRoot, framework: runner, language: language, command: command, args: args, platform: detectedPlatform, tracerVersion: version, tracerLabel: label, - executor: &ext.DefaultCommandExecutor{}, startIntake: func(directory string) (localIntake, error) { return intake.Start(directory) }}, nil + executor: &ext.DefaultCommandExecutor{}, startIntake: func(directory string) (localIntake, error) { return intake.Start(directory) }, nodeVersion: currentNodeVersion}, nil } func displayName(name string) string { @@ -403,8 +404,9 @@ func testEnvironment(intakeURL, sessionID string) map[string]string { func (t *Testdrive) environment(path, intakeURL, sessionID string) map[string]string { env := testEnvironment(intakeURL, sessionID) - if t.language == "javascript" { - maps.Copy(env, javascriptEnvironment(path)) + switch t.language { + case "javascript": + maps.Copy(env, t.javascriptEnvironment(path)) } return env } From c902d69b5c133a50f6fb326ebdb331797f2a87ed Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 30 Sep 2026 11:51:50 +0200 Subject: [PATCH 07/35] Remove Cucumber impacted-test workaround --- internal/testdrive/frameworks_integration_test.go | 2 +- internal/testdrive/javascript.go | 6 ------ internal/testdrive/multiframework_test.go | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index 5767f33a..82ff1179 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -36,7 +36,7 @@ func TestPublicFrameworkTestdrives(t *testing.T) { {"mocha", `{"scripts":{"test":"mocha"},"devDependencies":{"mocha":"11.7.5"}}`, "npm test", map[string]string{"test/one.js": `const assert = require('node:assert'); it('adds', () => assert.equal(1+1,2));`}}, {"vitest", `{"type":"module","scripts":{"test":"vitest run"},"devDependencies":{"vitest":"3.2.4"}}`, "npm test", map[string]string{"one.test.js": `import {test,expect} from 'vitest'; test('adds', () => expect(1+1).toBe(2));`}}, {"playwright", `{"scripts":{"test":"playwright test"},"devDependencies":{"@playwright/test":"1.55.1"}}`, "npm test", map[string]string{"one.spec.js": `const {test,expect} = require('@playwright/test'); test('adds', () => expect(1+1).toBe(2));`}}, - {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Background:\n Given addition works\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, + {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, } for _, fixture := range fixtures { t.Run(fixture.name, func(t *testing.T) { diff --git a/internal/testdrive/javascript.go b/internal/testdrive/javascript.go index b220b844..62a27f3c 100644 --- a/internal/testdrive/javascript.go +++ b/internal/testdrive/javascript.go @@ -101,11 +101,5 @@ func (t *Testdrive) javascriptEnvironment(path string) map[string]string { register := absoluteFileURL(filepath.Join(filepath.Dir(filepath.Dir(path)), "register.js")) env["NODE_OPTIONS"] += " --import " + strconv.Quote(register) } - // dd-trace 6.15.0 impacted-test detection dereferences scenario.id on - // Background/Rule nodes. Basic Cucumber reporting works with it off. - if t.framework.Name() == "cucumber" { - env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"] = "false" - } - return env } diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 4c1e70cb..fcf34947 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -53,7 +53,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { require.Contains(t, output.String(), "Tests with coverage: 0 / 1") require.Equal(t, "ddtest-testdrive", executor.env["DD_API_KEY"]) if name == "cucumber" { - require.Equal(t, "false", executor.env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"]) + require.Equal(t, "true", executor.env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"]) } switch run.language { case "javascript": From 10cb101e56924d2cbeca3cfb61b1bc467ddcf6b8 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 30 Sep 2026 13:09:22 +0200 Subject: [PATCH 08/35] Skip absent JavaScript ESM preload for reused tracers --- internal/testdrive/javascript.go | 6 ++++-- internal/testdrive/multiframework_test.go | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/testdrive/javascript.go b/internal/testdrive/javascript.go index 62a27f3c..db3939bb 100644 --- a/internal/testdrive/javascript.go +++ b/internal/testdrive/javascript.go @@ -98,8 +98,10 @@ func (t *Testdrive) javascriptEnvironment(path string) map[string]string { version = t.nodeVersion() } if supportsNodeImport(version) { - register := absoluteFileURL(filepath.Join(filepath.Dir(filepath.Dir(path)), "register.js")) - env["NODE_OPTIONS"] += " --import " + strconv.Quote(register) + registerPath := filepath.Join(filepath.Dir(filepath.Dir(path)), "register.js") + if info, err := os.Stat(registerPath); err == nil && info.Mode().IsRegular() { + env["NODE_OPTIONS"] += " --import " + strconv.Quote(absoluteFileURL(registerPath)) + } } return env } diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index fcf34947..211fc42c 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -57,7 +57,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { } switch run.language { case "javascript": - require.Contains(t, executor.env["NODE_OPTIONS"], "--import") + require.NotContains(t, executor.env["NODE_OPTIONS"], "--import") case "python": require.NotEmpty(t, executor.env["PYTHONPATH"]) require.Contains(t, executor.env["PYTEST_ADDOPTS"], "--ddtrace") @@ -73,6 +73,19 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { } } +func TestJavaScriptEnvironmentImportsExistingRegister(t *testing.T) { + t.Setenv("NODE_OPTIONS", "") + packageDir := filepath.Join(t.TempDir(), "dd-trace") + require.NoError(t, os.MkdirAll(filepath.Join(packageDir, "ci"), 0755)) + register := filepath.Join(packageDir, "register.js") + require.NoError(t, os.WriteFile(register, nil, 0644)) + run := &Testdrive{nodeVersion: func() string { return "v22.0.0" }} + preload := filepath.Join(packageDir, "ci", "init.js") + require.Contains(t, run.javascriptEnvironment(preload)["NODE_OPTIONS"], " --import ") + require.NoError(t, os.Remove(register)) + require.NotContains(t, run.javascriptEnvironment(preload)["NODE_OPTIONS"], "--import") +} + func TestSupportsNodeImport(t *testing.T) { for version, want := range map[string]bool{ "v18.17.1": false, From 4b59c647bb7cc6b9c8bff43671bb9aec762c63d3 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 30 Sep 2026 14:22:03 +0200 Subject: [PATCH 09/35] Flag missing coverage and advise Cucumber nyc setup --- internal/testdrive/intake/findings.go | 2 + internal/testdrive/intake/troubleshooting.go | 62 +++++++++++++++++++ .../testdrive/intake/troubleshooting_test.go | 54 ++++++++++++++++ internal/testdrive/report.go | 17 +++++ internal/testdrive/report.html | 26 +++++++- internal/testdrive/report_test.go | 26 ++++++++ internal/testdrive/testdrive.go | 4 ++ 7 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 internal/testdrive/intake/troubleshooting.go create mode 100644 internal/testdrive/intake/troubleshooting_test.go diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go index 56256dd8..cebec56e 100644 --- a/internal/testdrive/intake/findings.go +++ b/internal/testdrive/intake/findings.go @@ -81,6 +81,7 @@ type SlowSuite struct { type Facts struct { ConfigurationErrors []string EmptyCoverageEntryCount int + MissingCoverage bool TestCount int TestEventCount int CoveredTestCount int @@ -116,6 +117,7 @@ func (s *Server) Facts() (Facts, error) { findings.SlowSuites, findings.SuiteDurationMedian = slowSuites(findings.Tests) findings.TestCount = len(findings.Tests) findings.CoveredTestCount = uniqueCoveredTestCount(tests, coverages) + findings.MissingCoverage = len(tests) > 0 && len(coverages) == 0 && emptyEntries == 0 findings.BroadCoverage, findings.CoveredFilesMedian = analyzeCoverage(tests, coverages, findings.CoverageLevel) findings.ConfigurationErrors, err = s.configurationErrors() return findings, err diff --git a/internal/testdrive/intake/troubleshooting.go b/internal/testdrive/intake/troubleshooting.go new file mode 100644 index 00000000..82b3609b --- /dev/null +++ b/internal/testdrive/intake/troubleshooting.go @@ -0,0 +1,62 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" +) + +const MissingCoverageFinding = "missing-coverage" + +// Advice explains a likely cause of a finding and a concrete next step. +type Advice struct { + Finding string + Title string + Text string + URL string +} + +// Troubleshoot returns advice only when the project provides evidence for it. +func Troubleshoot(repositoryRoot, framework string, facts Facts) []Advice { + if !facts.MissingCoverage || !strings.EqualFold(framework, "cucumber") || !nycAbsent(repositoryRoot) { + return nil + } + return []Advice{{ + Finding: MissingCoverageFinding, + Title: "Cucumber needs nyc for coverage", + Text: "This project does not declare or install nyc locally. Add it with npm install --save-dev nyc, then run cucumber-js through nyc (for example, nyc cucumber-js) and repeat the testdrive.", + URL: "https://docs.datadoghq.com/tests/test_impact_analysis/setup/javascript/", + }} +} + +func nycAbsent(repositoryRoot string) bool { + data, err := os.ReadFile(filepath.Join(repositoryRoot, "package.json")) + if err != nil { + return false + } + var manifest struct { + Dependencies map[string]json.RawMessage `json:"dependencies"` + DevDependencies map[string]json.RawMessage `json:"devDependencies"` + OptionalDependencies map[string]json.RawMessage `json:"optionalDependencies"` + PeerDependencies map[string]json.RawMessage `json:"peerDependencies"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return false + } + for _, dependencies := range []map[string]json.RawMessage{ + manifest.Dependencies, manifest.DevDependencies, manifest.OptionalDependencies, manifest.PeerDependencies, + } { + if _, exists := dependencies["nyc"]; exists { + return false + } + } + _, err = os.Stat(filepath.Join(repositoryRoot, "node_modules", ".bin", "nyc")) + return errors.Is(err, os.ErrNotExist) +} diff --git a/internal/testdrive/intake/troubleshooting_test.go b/internal/testdrive/intake/troubleshooting_test.go new file mode 100644 index 00000000..ae8b17f3 --- /dev/null +++ b/internal/testdrive/intake/troubleshooting_test.go @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tinylib/msgp/msgp" +) + +func TestMissingCoverageFindingRequiresTestEventsAndNoCoveragePayload(t *testing.T) { + events := msgp.AppendMapHeader(nil, 1) + events = msgp.AppendString(events, "events") + events = msgp.AppendArrayHeader(events, 1) + events = appendEvent(events, "test", 10, 20, 100) + + server := serverWithCoverage(t, events) + server.requests = server.requests[:1] + facts, err := server.Facts() + require.NoError(t, err) + require.True(t, facts.MissingCoverage) + + server = serverWithCoverage(t, events, appendCoverage(nil, 10, 20, 100, "src/greet.js")) + facts, err = server.Facts() + require.NoError(t, err) + require.False(t, facts.MissingCoverage) + + server = serverWithCoverage(t, events, appendCoverage(nil, 10, 20, 100)) + facts, err = server.Facts() + require.NoError(t, err) + require.False(t, facts.MissingCoverage) // Empty coverage has its own finding. +} + +func TestTroubleshootCucumberWithoutNYC(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"devDependencies":{"@cucumber/cucumber":"13.2.1"}}`), 0644)) + facts := Facts{MissingCoverage: true} + + advice := Troubleshoot(root, "Cucumber", facts) + require.Len(t, advice, 1) + require.Equal(t, MissingCoverageFinding, advice[0].Finding) + require.Contains(t, advice[0].Text, "npm install --save-dev nyc") + require.Empty(t, Troubleshoot(root, "Cucumber", Facts{})) + require.Empty(t, Troubleshoot(root, "Jest", facts)) + + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"devDependencies":{"@cucumber/cucumber":"13.2.1","nyc":"17.0.0"}}`), 0644)) + require.Empty(t, Troubleshoot(root, "Cucumber", facts)) +} diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index 48a418e8..c61e0b4a 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -28,6 +28,7 @@ type reportCard struct { Title string Count int Context string + Advices []intake.Advice Tests []reportTest Suites []reportSuite Coverages []reportCoverage @@ -220,6 +221,12 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo Context: fmt.Sprintf("%d coverage entries had an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.", findings.EmptyCoverageEntryCount), }) } + if findings.MissingCoverage { + model.Cards = append(model.Cards, reportCard{ + Kind: intake.MissingCoverageFinding, Title: "Coverage not reported", Count: 1, + Context: "Test events arrived, but no code coverage was reported. Test Impact Analysis cannot map these tests to changed files.", + }) + } if len(findings.FailedTests) > 0 { model.Cards = append(model.Cards, reportCard{ Kind: "failed", Title: "Failed tests", Context: "Inspect the errors and source behind each failure.", Count: len(findings.FailedTests), @@ -260,6 +267,9 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo }) } count := len(findings.ConfigurationErrors) + findings.EmptyCoverageEntryCount + if findings.MissingCoverage { + count++ + } for _, size := range []int{ len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.SlowSuites), len(findings.BroadCoverage), } { @@ -271,6 +281,13 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo if len(findings.ConfigurationErrors) > 0 { model.Summary += " Tracer configuration errors: " + strings.Join(findings.ConfigurationErrors, ", ") + ". Inspect the captured traffic and test output." } + for _, advice := range intake.Troubleshoot(repositoryRoot, info.Framework, findings) { + for i := range model.Cards { + if model.Cards[i].Kind == advice.Finding { + model.Cards[i].Advices = append(model.Cards[i].Advices, advice) + } + } + } return model } diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index 3cb8a706..bfbe6624 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -546,7 +546,8 @@

--card-accent: var(--red); } .problem-card[data-kind="flaky"], - .problem-card[data-kind="slow"] { + .problem-card[data-kind="slow"], + .problem-card[data-kind="missing-coverage"] { --card-accent: var(--amber); } .card-button { @@ -616,6 +617,17 @@

justify-content: space-between; background: var(--surface); } + .finding-explanation, + .finding-advice { + padding: 16px 20px; + border-top: 1px solid var(--line); + } + .finding-advice { + background: var(--raised); + } + .finding-advice p { + margin: 8px 0; + } h2 { font-size: 16px; margin: 0; @@ -1208,6 +1220,8 @@

Test report

{{ template "icon-arrow-repeat" }} {{ else if eq .Kind "slow" }} {{ template "icon-stopwatch" }} + {{ else if eq .Kind "missing-coverage" }} + {{ template "icon-exclamation-circle" }} {{ else if eq .Kind "error" }} {{ template "icon-exclamation-circle" }} {{ else }} @@ -1234,6 +1248,16 @@

Test report

{{ .Title }} · {{ .Count }}

+ {{ if eq .Kind "missing-coverage" }} +

{{ .Context }}

+ {{ end }} + {{ range .Advices }} + + {{ end }} {{ range .Tests }}
diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index b9d88e0f..38440d0f 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -24,6 +24,7 @@ func TestReportCountsIndividualFindings(t *testing.T) { want string }{ {"single", intake.Facts{TestEventCount: 1, FailedTests: []intake.Test{{Name: "failed"}}}, "1 finding."}, + {"missing coverage", intake.Facts{TestEventCount: 1, TestCount: 1, MissingCoverage: true}, "1 finding."}, {"multiple in one card", intake.Facts{TestEventCount: 2, FailedTests: []intake.Test{{Name: "one"}, {Name: "two"}}}, "2 findings."}, {"empty coverage without events", intake.Facts{EmptyCoverageEntryCount: 2}, "2 findings."}, {"configuration error without events", intake.Facts{ConfigurationErrors: []string{"skippable_tests"}}, "1 finding."}, @@ -50,6 +51,31 @@ func TestReportCountsIndividualFindings(t *testing.T) { } } +func TestReportShowsCucumberCoverageTroubleshootingWithFinding(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"devDependencies":{"@cucumber/cucumber":"13.2.1"}}`), 0644); err != nil { + t.Fatal(err) + } + facts := intake.Facts{TestEventCount: 2, TestCount: 2, MissingCoverage: true} + model := buildReport(root, facts, false, reportRuntime{Framework: "Cucumber"}) + if model.Summary != "1 finding." || len(model.Cards) != 1 || len(model.Cards[0].Advices) != 1 { + t.Fatalf("missing coverage advice: summary=%q cards=%+v", model.Summary, model.Cards) + } + path, err := writeReport(root, t.TempDir(), facts, false, reportRuntime{Framework: "Cucumber"}) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, expected := range []string{"Coverage not reported", "Cucumber needs nyc for coverage", "npm install --save-dev nyc"} { + if !strings.Contains(string(data), expected) { + t.Fatalf("report does not show %q", expected) + } + } +} + func TestReportKeepsPythonAndRubySourceEscaped(t *testing.T) { for _, extension := range []string{".py", ".rb"} { t.Run(extension, func(t *testing.T) { diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index 5561571c..cee45471 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -301,6 +301,10 @@ func writeFindings(output io.Writer, findings intake.Facts) { _, _ = fmt.Fprintf(output, "Tracer configuration errors: %s. Inspect the captured traffic and test output.\n", strings.Join(findings.ConfigurationErrors, ", ")) } count := len(findings.ConfigurationErrors) + if findings.MissingCoverage { + count++ + _, _ = fmt.Fprintln(output, "Coverage not reported: test events arrived, but no code coverage was reported. Test Impact Analysis cannot map these tests to changed files.") + } if findings.EmptyCoverageEntryCount > 0 { count += findings.EmptyCoverageEntryCount _, _ = fmt.Fprintf(output, "Tracer error: received %d coverage entries with an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.\n", findings.EmptyCoverageEntryCount) From 45836820ce2091e0cdff44bf7c18a9d4f9078b66 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:14:47 +0200 Subject: [PATCH 10/35] Support Cypress with preserved project hooks --- internal/testdrive/cypress.go | 104 ++++++++++++++++++ .../testdrive/frameworks_integration_test.go | 1 + internal/testdrive/multiframework_test.go | 20 +++- 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 internal/testdrive/cypress.go diff --git a/internal/testdrive/cypress.go b/internal/testdrive/cypress.go new file mode 100644 index 00000000..ed210d09 --- /dev/null +++ b/internal/testdrive/cypress.go @@ -0,0 +1,104 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// prepareCypress wraps configuration inside the session; project config and +// support files are neither edited nor replaced on disk. +func prepareCypress(root, directory, preload string, args []string) ([]string, error) { + var config string + clean := make([]string, 0, len(args)+2) + for i := 0; i < len(args); i++ { + if args[i] == "--config-file" { + if i+1 == len(args) { + return nil, fmt.Errorf("--config-file requires a path") + } + i++ + config = args[i] + continue + } + if strings.HasPrefix(args[i], "--config-file=") { + config = strings.TrimPrefix(args[i], "--config-file=") + continue + } + clean = append(clean, args[i]) + } + if config == "" { + for _, name := range []string{"cypress.config.ts", "cypress.config.js", "cypress.config.mjs", "cypress.config.cjs"} { + if _, err := os.Stat(filepath.Join(root, name)); err == nil { + config = name + break + } + } + } + if config == "" { + return nil, fmt.Errorf("cypress needs a config file; provide its path with --command '... --config-file path'") + } + if !filepath.IsAbs(config) { + config = filepath.Join(root, config) + } + values, _ := json.Marshal(map[string]string{"root": root, "config": config, "directory": directory, "tracer": filepath.Dir(filepath.Dir(preload))}) + wrapper := "const options = " + string(values) + ";\n" + cypressWrapper + path := filepath.Join(directory, "cypress.config.cjs") + if err := os.WriteFile(path, []byte(wrapper), 0600); err != nil { + return nil, err + } + return append(clean, "--config-file", path), nil +} + +const cypressWrapper = ` +const fs = require('node:fs'); +const path = require('node:path'); +const { pathToFileURL } = require('node:url'); +// Cypress starts the config process in the wrapper's directory. Keep relative +// filesystem operations in the customer's config and hooks rooted in the project. +process.chdir(options.root); +module.exports = (async () => { + let original; + try { original = require(options.config); } + catch (error) { + if (error.code !== 'ERR_REQUIRE_ESM') throw error; + original = await import(pathToFileURL(options.config).href); + } + original = original.default || original; + const config = { ...original }; + for (const type of ['e2e', 'component']) { + if (!original[type]) continue; + const setup = original[type].setupNodeEvents; + config[type] = { ...original[type], async setupNodeEvents(on, resolved) { + // Compose hooks so adding instrumentation cannot discard customer hooks. + const handlers = new Map(); + const collect = (event, handler) => { + if (event === 'task') { handlers.set(event, { ...(handlers.get(event) || {}), ...handler }); return; } + const previous = handlers.get(event); + handlers.set(event, previous ? async (...args) => { + const first = await previous(...args); + const second = await handler(...args); + return second === undefined ? first : second; + } : handler); + }; + if (setup) resolved = (await setup(collect, resolved)) || resolved; + const support = path.join(options.directory, type + '-support.cjs'); + let contents = 'require(' + JSON.stringify(path.join(options.tracer, 'ci/cypress/support')) + ');\n'; + if (resolved.supportFile) contents += 'require(' + JSON.stringify(path.resolve(resolved.projectRoot, resolved.supportFile)) + ');\n'; + fs.writeFileSync(support, contents); + resolved.supportFile = support; + const instrument = require(path.join(options.tracer, 'ci/cypress/plugin')); + resolved = (await instrument(collect, resolved)) || resolved; + for (const [event, handler] of handlers) on(event, handler); + return resolved; + }}; + } + return config; +})(); +` diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index 82ff1179..d1ca7d93 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -37,6 +37,7 @@ func TestPublicFrameworkTestdrives(t *testing.T) { {"vitest", `{"type":"module","scripts":{"test":"vitest run"},"devDependencies":{"vitest":"3.2.4"}}`, "npm test", map[string]string{"one.test.js": `import {test,expect} from 'vitest'; test('adds', () => expect(1+1).toBe(2));`}}, {"playwright", `{"scripts":{"test":"playwright test"},"devDependencies":{"@playwright/test":"1.55.1"}}`, "npm test", map[string]string{"one.spec.js": `const {test,expect} = require('@playwright/test'); test('adds', () => expect(1+1).toBe(2));`}}, {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, + {"cypress", `{"scripts":{"test":"cypress run"},"devDependencies":{"cypress":"15.1.0"}}`, "npm test", map[string]string{"cypress.config.js": `module.exports={e2e:{supportFile:false,setupNodeEvents(on,config){on('task',{answer:()=>42});on('after:run',()=>{require('node:fs').writeFileSync('original-hook.txt','ran');});return config;}}};`, "cypress/e2e/one.cy.js": `it('preserves hooks', () => { cy.task('answer').should('equal',42); });`}}, } for _, fixture := range fixtures { t.Run(fixture.name, func(t *testing.T) { diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 211fc42c..551b3206 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -17,7 +17,7 @@ import ( ) func TestPrepareAllSupportedFrameworks(t *testing.T) { - for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber"} { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress"} { t.Run(name, func(t *testing.T) { root := t.TempDir() switch name { @@ -116,3 +116,21 @@ func TestPrepareRequiresSelectionForMultipleFrameworks(t *testing.T) { _, err = Prepare("latest") require.ErrorContains(t, err, "unsupported framework") } + +func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { + root := t.TempDir() + session := t.TempDir() + config := `module.exports={e2e:{supportFile:false}}` + requireWriteFile(t, filepath.Join(root, "custom.cjs"), config) + args, err := prepareCypress(root, session, "/tracer/ci/init.js", []string{"run", "--config-file=custom.cjs", "--browser", "chrome"}) + require.NoError(t, err) + require.Equal(t, []string{"run", "--browser", "chrome", "--config-file", filepath.Join(session, "cypress.config.cjs")}, args) + contents, err := os.ReadFile(filepath.Join(root, "custom.cjs")) + require.NoError(t, err) + require.Equal(t, config, string(contents)) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.cjs")) + require.NoError(t, err) + require.Contains(t, string(wrapper), filepath.Join(root, "custom.cjs")) + _, err = prepareCypress(root, session, "/tracer/ci/init.js", []string{"--config-file"}) + require.ErrorContains(t, err, "requires a path") +} From a61832706a326cb804a39c37a631d6e9204f1d65 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 09:14:02 +0200 Subject: [PATCH 11/35] Preserve Cypress project configuration --- internal/testdrive/cypress.go | 154 +++++++++++++++++----- internal/testdrive/multiframework_test.go | 37 +++++- internal/testdrive/testdrive.go | 15 ++- 3 files changed, 171 insertions(+), 35 deletions(-) diff --git a/internal/testdrive/cypress.go b/internal/testdrive/cypress.go index ed210d09..6173acc6 100644 --- a/internal/testdrive/cypress.go +++ b/internal/testdrive/cypress.go @@ -11,15 +11,26 @@ import ( "os" "path/filepath" "strings" + + "github.com/kballard/go-shellquote" ) // prepareCypress wraps configuration inside the session; project config and // support files are neither edited nor replaced on disk. -func prepareCypress(root, directory, preload string, args []string) ([]string, error) { +func prepareCypress(root, directory, preload, command string, args []string) ([]string, error) { + inspectionArgs := cypressInspectionArgs(root, command, args) + projectRoot := root + if project := optionValue(inspectionArgs, "--project", "-P"); project != "" { + if filepath.IsAbs(project) { + projectRoot = project + } else { + projectRoot = filepath.Join(root, project) + } + } var config string clean := make([]string, 0, len(args)+2) for i := 0; i < len(args); i++ { - if args[i] == "--config-file" { + if args[i] == "--config-file" || args[i] == "-C" { if i+1 == len(args) { return nil, fmt.Errorf("--config-file requires a path") } @@ -27,55 +38,138 @@ func prepareCypress(root, directory, preload string, args []string) ([]string, e config = args[i] continue } - if strings.HasPrefix(args[i], "--config-file=") { - config = strings.TrimPrefix(args[i], "--config-file=") + if strings.HasPrefix(args[i], "--config-file=") || strings.HasPrefix(args[i], "-C=") { + _, config, _ = strings.Cut(args[i], "=") continue } clean = append(clean, args[i]) } + if config == "" { + config = optionValue(inspectionArgs, "--config-file", "-C") + } + if config == "false" { + config = "" + } if config == "" { for _, name := range []string{"cypress.config.ts", "cypress.config.js", "cypress.config.mjs", "cypress.config.cjs"} { - if _, err := os.Stat(filepath.Join(root, name)); err == nil { + if _, err := os.Stat(filepath.Join(projectRoot, name)); err == nil { config = name break } } } - if config == "" { - return nil, fmt.Errorf("cypress needs a config file; provide its path with --command '... --config-file path'") + if config != "" && !filepath.IsAbs(config) { + config = filepath.Join(projectRoot, config) + } + testingType := "e2e" + if optionPresent(inspectionArgs, "--component") { + testingType = "component" } - if !filepath.IsAbs(config) { - config = filepath.Join(root, config) + tracerRoot := filepath.Dir(filepath.Dir(preload)) + values, _ := json.Marshal(map[string]string{"root": projectRoot, "directory": directory, "tracer": tracerRoot, "testingType": testingType}) + configImport := "const originalImport = {};\n" + if config != "" { + encodedConfig, _ := json.Marshal(config) + configImport = "import originalImport from " + string(encodedConfig) + ";\n" } - values, _ := json.Marshal(map[string]string{"root": root, "config": config, "directory": directory, "tracer": filepath.Dir(filepath.Dir(preload))}) - wrapper := "const options = " + string(values) + ";\n" + cypressWrapper - path := filepath.Join(directory, "cypress.config.cjs") + encodedPlugin, _ := json.Marshal(filepath.Join(tracerRoot, "ci", "cypress", "plugin")) + wrapper := "import fs from 'node:fs';\nimport path from 'node:path';\n" + configImport + + "import instrumentImport from " + string(encodedPlugin) + ";\nconst options = " + string(values) + ";\n" + cypressWrapper + path := filepath.Join(directory, "cypress.config.ts") if err := os.WriteFile(path, []byte(wrapper), 0600); err != nil { return nil, err } return append(clean, "--config-file", path), nil } +func optionValue(args []string, options ...string) string { + value := "" + for index := 0; index < len(args); index++ { + for _, option := range options { + if args[index] == option && index+1 < len(args) { + value = args[index+1] + index++ + break + } + if candidate, found := strings.CutPrefix(args[index], option+"="); found { + value = candidate + break + } + } + } + return value +} + +func optionPresent(args []string, option string) bool { + for _, arg := range args { + if arg == option || strings.HasPrefix(arg, option+"=") { + return true + } + } + return false +} + +func cypressInspectionArgs(root, command string, args []string) []string { + for _, arg := range append([]string{command}, args...) { + if strings.Contains(strings.ToLower(filepath.Base(arg)), "cypress") { + return args + } + } + base := strings.ToLower(filepath.Base(command)) + if base != "npm" && base != "yarn" && base != "pnpm" && base != "bun" { + return args + } + script := "" + if len(args) > 0 && args[0] == "test" { + script = "test" + } else if len(args) > 1 && (args[0] == "run" || args[0] == "run-script") { + script = args[1] + } + if script == "" { + return args + } + contents, err := os.ReadFile(filepath.Join(root, "package.json")) + if err != nil { + return args + } + var manifest struct { + Scripts map[string]string `json:"scripts"` + } + if json.Unmarshal(contents, &manifest) != nil { + return args + } + expanded, err := shellquote.Split(manifest.Scripts[script]) + if err != nil { + return args + } + if separator := slicesIndex(args, "--"); separator >= 0 { + expanded = append(expanded, args[separator+1:]...) + } + return expanded +} + +func slicesIndex(values []string, value string) int { + for index, candidate := range values { + if candidate == value { + return index + } + } + return -1 +} + const cypressWrapper = ` -const fs = require('node:fs'); -const path = require('node:path'); -const { pathToFileURL } = require('node:url'); // Cypress starts the config process in the wrapper's directory. Keep relative // filesystem operations in the customer's config and hooks rooted in the project. process.chdir(options.root); -module.exports = (async () => { - let original; - try { original = require(options.config); } - catch (error) { - if (error.code !== 'ERR_REQUIRE_ESM') throw error; - original = await import(pathToFileURL(options.config).href); - } - original = original.default || original; +export default (async () => { + const original = originalImport.default || originalImport; const config = { ...original }; - for (const type of ['e2e', 'component']) { - if (!original[type]) continue; - const setup = original[type].setupNodeEvents; - config[type] = { ...original[type], async setupNodeEvents(on, resolved) { + const types = new Set(['e2e', 'component'].filter(type => original[type])); + types.add(options.testingType); + for (const type of types) { + const originalType = original[type] || {}; + const setup = originalType.setupNodeEvents; + config[type] = { ...originalType, async setupNodeEvents(on, resolved) { // Compose hooks so adding instrumentation cannot discard customer hooks. const handlers = new Map(); const collect = (event, handler) => { @@ -87,13 +181,13 @@ module.exports = (async () => { return second === undefined ? first : second; } : handler); }; - if (setup) resolved = (await setup(collect, resolved)) || resolved; + if (setup) resolved = { ...resolved, ...((await setup(collect, resolved)) || {}) }; const support = path.join(options.directory, type + '-support.cjs'); let contents = 'require(' + JSON.stringify(path.join(options.tracer, 'ci/cypress/support')) + ');\n'; - if (resolved.supportFile) contents += 'require(' + JSON.stringify(path.resolve(resolved.projectRoot, resolved.supportFile)) + ');\n'; + if (resolved.supportFile) contents += 'require(' + JSON.stringify(path.resolve(resolved.projectRoot || options.root, resolved.supportFile)) + ');\n'; fs.writeFileSync(support, contents); resolved.supportFile = support; - const instrument = require(path.join(options.tracer, 'ci/cypress/plugin')); + const instrument = instrumentImport.default || instrumentImport; resolved = (await instrument(collect, resolved)) || resolved; for (const [event, handler] of handlers) on(event, handler); return resolved; diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 551b3206..01fa01aa 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -122,15 +122,44 @@ func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { session := t.TempDir() config := `module.exports={e2e:{supportFile:false}}` requireWriteFile(t, filepath.Join(root, "custom.cjs"), config) - args, err := prepareCypress(root, session, "/tracer/ci/init.js", []string{"run", "--config-file=custom.cjs", "--browser", "chrome"}) + args, err := prepareCypress(root, session, "/tracer/ci/init.js", "cypress", []string{"run", "--config-file=custom.cjs", "--browser", "chrome"}) require.NoError(t, err) - require.Equal(t, []string{"run", "--browser", "chrome", "--config-file", filepath.Join(session, "cypress.config.cjs")}, args) + require.Equal(t, []string{"run", "--browser", "chrome", "--config-file", filepath.Join(session, "cypress.config.ts")}, args) contents, err := os.ReadFile(filepath.Join(root, "custom.cjs")) require.NoError(t, err) require.Equal(t, config, string(contents)) - wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.cjs")) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.ts")) require.NoError(t, err) require.Contains(t, string(wrapper), filepath.Join(root, "custom.cjs")) - _, err = prepareCypress(root, session, "/tracer/ci/init.js", []string{"--config-file"}) + _, err = prepareCypress(root, session, "/tracer/ci/init.js", "cypress", []string{"--config-file"}) require.ErrorContains(t, err, "requires a path") } + +func TestCypressWrapperReadsPackageScriptProjectAndConfig(t *testing.T) { + root := t.TempDir() + session := t.TempDir() + project := filepath.Join(root, "apps", "web") + require.NoError(t, os.MkdirAll(project, 0755)) + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"cypress run --project apps/web -C custom.ts"}}`) + requireWriteFile(t, filepath.Join(project, "custom.ts"), `export default {}`) + + args, err := prepareCypress(root, session, "/tracer/ci/init.js", "npm", []string{"test", "--"}) + require.NoError(t, err) + require.Equal(t, []string{"test", "--", "--config-file", filepath.Join(session, "cypress.config.ts")}, args) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.ts")) + require.NoError(t, err) + require.Contains(t, string(wrapper), filepath.Join(project, "custom.ts")) + require.Contains(t, string(wrapper), `"root":"`+project+`"`) +} + +func TestCypressWrapperSupportsConfigFileFalseAndDefaultE2E(t *testing.T) { + root := t.TempDir() + session := t.TempDir() + args, err := prepareCypress(root, session, "/tracer/ci/init.js", "cypress", []string{"run", "-C", "false"}) + require.NoError(t, err) + require.Equal(t, []string{"run", "--config-file", filepath.Join(session, "cypress.config.ts")}, args) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.ts")) + require.NoError(t, err) + require.Contains(t, string(wrapper), `types.add(options.testingType)`) + require.Contains(t, string(wrapper), `const originalImport = {}`) +} diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index cee45471..cd6728d0 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -14,6 +14,7 @@ import ( "maps" "os" "path/filepath" + "slices" "strings" "time" @@ -79,7 +80,7 @@ func Prepare(version string) (*Testdrive, error) { } language := detectedPlatform.Name() switch runner.Name() { - case "jest", "mocha", "vitest", "playwright", "cucumber": + case "jest", "mocha", "vitest", "playwright", "cucumber", "cypress": default: return nil, fmt.Errorf("testdrive does not yet support %s", runner.Name()) } @@ -134,6 +135,9 @@ func (t *Testdrive) Preview(output io.Writer) { _, _ = fmt.Fprintf(output, " - install %s: %s\n", t.tracerLabel, shellquote.Join(install...)) } + if t.framework.Name() == "cypress" { + _, _ = fmt.Fprintf(output, " - create Cypress config/support wrappers in %s and preserve existing hooks\n", directory) + } _, _ = fmt.Fprintf(output, " - run: %s\n", shellquote.Join(append([]string{command}, args...)...)) _, _ = fmt.Fprintln(output) patterns := map[string][]string{ @@ -204,6 +208,15 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { _, _ = fmt.Fprintf(output, "Running %s...\n", shellquote.Join(append([]string{command}, args...)...)) env := t.environment(installation.Path, server.URL(), session.ID()) maps.Copy(env, installation.Env) + if t.framework.Name() == "cypress" { + if command == "npm" && !slices.Contains(args, "--") { + args = append(slices.Clone(args), "--") + } + args, err = prepareCypress(t.repositoryRoot, session.Directory(), installation.Path, command, args) + if err != nil { + return err + } + } testOutput, testErr := t.executor.CombinedOutput(ctx, command, args, env) From 2f839c86282353f01cb747a4f6edb6b97a3076a6 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:15:14 +0200 Subject: [PATCH 12/35] Support isolated pytest testdrives --- .../testdrive/frameworks_integration_test.go | 1 + internal/testdrive/multiframework_test.go | 12 ++++++- internal/testdrive/tracer/languages_test.go | 29 +++++++++++++++ internal/testdrive/tracer/python.go | 35 +++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 internal/testdrive/tracer/languages_test.go create mode 100644 internal/testdrive/tracer/python.go diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index d1ca7d93..bedaf9f3 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -38,6 +38,7 @@ func TestPublicFrameworkTestdrives(t *testing.T) { {"playwright", `{"scripts":{"test":"playwright test"},"devDependencies":{"@playwright/test":"1.55.1"}}`, "npm test", map[string]string{"one.spec.js": `const {test,expect} = require('@playwright/test'); test('adds', () => expect(1+1).toBe(2));`}}, {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, {"cypress", `{"scripts":{"test":"cypress run"},"devDependencies":{"cypress":"15.1.0"}}`, "npm test", map[string]string{"cypress.config.js": `module.exports={e2e:{supportFile:false,setupNodeEvents(on,config){on('task',{answer:()=>42});on('after:run',()=>{require('node:fs').writeFileSync('original-hook.txt','ran');});return config;}}};`, "cypress/e2e/one.cy.js": `it('preserves hooks', () => { cy.task('answer').should('equal',42); });`}}, + {"pytest", "", "python -m pytest", map[string]string{"requirements.txt": "pytest==8.4.2\n", "pytest.ini": "[pytest]\n", "test_one.py": "def test_adds():\n assert 1 + 1 == 2\n"}}, } for _, fixture := range fixtures { t.Run(fixture.name, func(t *testing.T) { diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 01fa01aa..869f0357 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -17,7 +17,7 @@ import ( ) func TestPrepareAllSupportedFrameworks(t *testing.T) { - for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress"} { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest"} { t.Run(name, func(t *testing.T) { root := t.TempDir() switch name { @@ -117,6 +117,16 @@ func TestPrepareRequiresSelectionForMultipleFrameworks(t *testing.T) { require.ErrorContains(t, err, "unsupported framework") } +func TestLanguageEnvironmentsPreserveCustomerOptions(t *testing.T) { + t.Setenv("PYTHONPATH", "/customer/modules") + t.Setenv("PYTEST_ADDOPTS", "-q") + t.Setenv("RUBYOPT", "-W0") + python := (&Testdrive{language: "python"}).environment("/session/python", "http://127.0.0.1:1234", "session") + require.Equal(t, "/session/python"+string(os.PathListSeparator)+"/customer/modules", python["PYTHONPATH"]) + require.Equal(t, "-q --ddtrace", python["PYTEST_ADDOPTS"]) + +} + func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { root := t.TempDir() session := t.TempDir() diff --git a/internal/testdrive/tracer/languages_test.go b/internal/testdrive/tracer/languages_test.go new file mode 100644 index 00000000..6e0da702 --- /dev/null +++ b/internal/testdrive/tracer/languages_test.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package tracer + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPythonInstallUsesSelectedInterpreterAndIsolatedTarget(t *testing.T) { + directory := t.TempDir() + executor := &fakeCommandExecutor{responses: []commandResponse{{}}} + installer := &Python{Interpreter: "/customer/venv/bin/python", executor: executor} + path, err := installer.Install(t.Context(), directory) + require.NoError(t, err) + require.Equal(t, filepath.Join(directory, "python"), path) + require.Equal(t, "/customer/venv/bin/python", executor.commands[0].name) + require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", path, "ddtrace==" + PythonVersion}, executor.commands[0].args) + executor = &fakeCommandExecutor{responses: []commandResponse{{output: []byte("pip unavailable"), err: errors.New("exit 1")}}} + installer.executor = executor + _, err = installer.Install(t.Context(), directory) + require.ErrorContains(t, err, "pip unavailable") +} diff --git a/internal/testdrive/tracer/python.go b/internal/testdrive/tracer/python.go new file mode 100644 index 00000000..593fdfa2 --- /dev/null +++ b/internal/testdrive/tracer/python.go @@ -0,0 +1,35 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package tracer + +import ( + "context" + "path/filepath" + + "github.com/DataDog/ddtest/internal/ext" +) + +const PythonVersion = "4.15.1" + +// Python installs into a private target, keeping the active interpreter and +// the customer's dependencies available when pytest runs. +type Python struct { + Interpreter string + executor commandExecutor +} + +func NewPython(interpreter string) *Python { + return &Python{Interpreter: interpreter, executor: &ext.DefaultCommandExecutor{}} +} + +func (p *Python) Install(ctx context.Context, directory string) (string, error) { + target := filepath.Join(directory, "python") + args := []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", target, "ddtrace==" + PythonVersion} + if output, err := p.executor.CombinedOutput(ctx, p.Interpreter, args, nil); err != nil { + return "", commandError("install ddtrace", output, err) + } + return target, nil +} From 7a040d0c955de219afbf736bbefadcdf955ac7ca Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 09:15:04 +0200 Subject: [PATCH 13/35] Use the pytest runner Python safely --- internal/testdrive/tracer/languages_test.go | 25 ++++++++- internal/testdrive/tracer/python.go | 60 +++++++++++++++++++-- 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/internal/testdrive/tracer/languages_test.go b/internal/testdrive/tracer/languages_test.go index 6e0da702..bc3c0006 100644 --- a/internal/testdrive/tracer/languages_test.go +++ b/internal/testdrive/tracer/languages_test.go @@ -7,6 +7,7 @@ package tracer import ( "errors" + "os" "path/filepath" "testing" @@ -16,14 +17,34 @@ import ( func TestPythonInstallUsesSelectedInterpreterAndIsolatedTarget(t *testing.T) { directory := t.TempDir() executor := &fakeCommandExecutor{responses: []commandResponse{{}}} - installer := &Python{Interpreter: "/customer/venv/bin/python", executor: executor} + installer := &Python{Interpreter: "/customer/venv/bin/python", command: "/customer/venv/bin/python", executor: executor} path, err := installer.Install(t.Context(), directory) require.NoError(t, err) require.Equal(t, filepath.Join(directory, "python"), path) require.Equal(t, "/customer/venv/bin/python", executor.commands[0].name) - require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", path, "ddtrace==" + PythonVersion}, executor.commands[0].args) + require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", filepath.Join(directory, "python-packages"), "ddtrace==" + PythonVersion}, executor.commands[0].args) + contents, err := os.ReadFile(filepath.Join(path, "sitecustomize.py")) + require.NoError(t, err) + require.Contains(t, string(contents), filepath.Join(directory, "python-packages")) executor = &fakeCommandExecutor{responses: []commandResponse{{output: []byte("pip unavailable"), err: errors.New("exit 1")}}} installer.executor = executor _, err = installer.Install(t.Context(), directory) require.ErrorContains(t, err, "pip unavailable") } +func TestNewPythonForCommandUsesRunnerInterpreter(t *testing.T) { + for _, test := range []struct { + command string + args []string + wantCommand string + wantPrefix []string + }{ + {command: "python3.12", args: []string{"-m", "pytest"}, wantCommand: "python3.12"}, + {command: ".venv/bin/pytest", wantCommand: filepath.Join(".venv", "bin", "python")}, + {command: "uv", args: []string{"run", "pytest"}, wantCommand: "uv", wantPrefix: []string{"run", "python"}}, + {command: "poetry", args: []string{"run", "pytest"}, wantCommand: "poetry", wantPrefix: []string{"run", "python"}}, + } { + installer := NewPythonForCommand(test.command, test.args, "python") + require.Equal(t, test.wantCommand, installer.command) + require.Equal(t, test.wantPrefix, installer.prefixArgs) + } +} diff --git a/internal/testdrive/tracer/python.go b/internal/testdrive/tracer/python.go index 593fdfa2..4af48826 100644 --- a/internal/testdrive/tracer/python.go +++ b/internal/testdrive/tracer/python.go @@ -7,9 +7,14 @@ package tracer import ( "context" + "encoding/json" + "fmt" + "os" "path/filepath" + "strings" "github.com/DataDog/ddtest/internal/ext" + "github.com/kballard/go-shellquote" ) const PythonVersion = "4.15.1" @@ -18,18 +23,63 @@ const PythonVersion = "4.15.1" // the customer's dependencies available when pytest runs. type Python struct { Interpreter string + command string + prefixArgs []string executor commandExecutor } func NewPython(interpreter string) *Python { - return &Python{Interpreter: interpreter, executor: &ext.DefaultCommandExecutor{}} + return &Python{Interpreter: interpreter, command: interpreter, executor: &ext.DefaultCommandExecutor{}} +} + +func NewPythonForCommand(command string, args []string, fallback string) *Python { + base := strings.ToLower(filepath.Base(strings.ReplaceAll(command, `\`, "/"))) + if isPythonExecutable(base) { + return NewPython(command) + } + if (base == "uv" || base == "poetry") && len(args) > 0 && args[0] == "run" { + python := NewPython(command) + python.prefixArgs = []string{"run", "python"} + python.Interpreter = shellquote.Join(command, "run", "python") + return python + } + if strings.HasPrefix(base, "pytest") && filepath.Dir(command) != "." { + interpreter := filepath.Join(filepath.Dir(command), "python") + if strings.HasSuffix(base, ".exe") { + interpreter += ".exe" + } + return NewPython(interpreter) + } + return NewPython(fallback) +} + +func isPythonExecutable(base string) bool { + base = strings.TrimSuffix(base, ".exe") + if !strings.HasPrefix(base, "python") { + return false + } + for _, character := range strings.TrimPrefix(base, "python") { + if (character < '0' || character > '9') && character != '.' { + return false + } + } + return true } func (p *Python) Install(ctx context.Context, directory string) (string, error) { - target := filepath.Join(directory, "python") - args := []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", target, "ddtrace==" + PythonVersion} - if output, err := p.executor.CombinedOutput(ctx, p.Interpreter, args, nil); err != nil { + target := filepath.Join(directory, "python-packages") + args := append(append([]string{}, p.prefixArgs...), "-m", "pip", "install", "--disable-pip-version-check", "--target", target, "ddtrace=="+PythonVersion) + if output, err := p.executor.CombinedOutput(ctx, p.command, args, nil); err != nil { return "", commandError("install ddtrace", output, err) } - return target, nil + bootstrap := filepath.Join(directory, "python") + if err := os.MkdirAll(bootstrap, 0755); err != nil { + return "", fmt.Errorf("create Python tracer bootstrap: %w", err) + } + encodedTarget, _ := json.Marshal(target) + contents := "import sys\nsys.path.append(" + string(encodedTarget) + ")\n" + if err := os.WriteFile(filepath.Join(bootstrap, "sitecustomize.py"), []byte(contents), 0600); err != nil { + return "", fmt.Errorf("write Python tracer bootstrap: %w", err) + } + return bootstrap, nil } From d3d8854e3b94903a31632c4eeaebc5f38ff07908 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 13:34:50 +0200 Subject: [PATCH 14/35] Select latest Python tracer or a release and Git revision --- internal/testdrive/tracer/languages_test.go | 26 +++++++++++++++-- internal/testdrive/tracer/python.go | 32 +++++++++++++-------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/internal/testdrive/tracer/languages_test.go b/internal/testdrive/tracer/languages_test.go index bc3c0006..60fef050 100644 --- a/internal/testdrive/tracer/languages_test.go +++ b/internal/testdrive/tracer/languages_test.go @@ -22,10 +22,11 @@ func TestPythonInstallUsesSelectedInterpreterAndIsolatedTarget(t *testing.T) { require.NoError(t, err) require.Equal(t, filepath.Join(directory, "python"), path) require.Equal(t, "/customer/venv/bin/python", executor.commands[0].name) - require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", filepath.Join(directory, "python-packages"), "ddtrace==" + PythonVersion}, executor.commands[0].args) + require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", filepath.Join(directory, "python-packages"), "ddtrace"}, executor.commands[0].args) contents, err := os.ReadFile(filepath.Join(path, "sitecustomize.py")) require.NoError(t, err) require.Contains(t, string(contents), filepath.Join(directory, "python-packages")) + require.Contains(t, string(contents), "sys.path.insert(0, ") executor = &fakeCommandExecutor{responses: []commandResponse{{output: []byte("pip unavailable"), err: errors.New("exit 1")}}} installer.executor = executor _, err = installer.Install(t.Context(), directory) @@ -43,8 +44,29 @@ func TestNewPythonForCommandUsesRunnerInterpreter(t *testing.T) { {command: "uv", args: []string{"run", "pytest"}, wantCommand: "uv", wantPrefix: []string{"run", "python"}}, {command: "poetry", args: []string{"run", "pytest"}, wantCommand: "poetry", wantPrefix: []string{"run", "python"}}, } { - installer := NewPythonForCommand(test.command, test.args, "python") + installer := NewPythonForCommand(test.command, test.args, "python", "latest") require.Equal(t, test.wantCommand, installer.command) require.Equal(t, test.wantPrefix, installer.prefixArgs) } } + +func TestPythonTracerVersions(t *testing.T) { + for _, tt := range []struct{ version, spec string }{ + {"", "ddtrace"}, {"latest", "ddtrace"}, {"4.15.1", "ddtrace==4.15.1"}, + {"4.16.0rc1", "ddtrace==4.16.0rc1"}, + {"git:abc1234", "ddtrace @ git+https://github.com/DataDog/dd-trace-py.git@abc1234"}, + } { + t.Run(tt.version, func(t *testing.T) { + executor := &fakeCommandExecutor{responses: []commandResponse{{}}} + installer := NewPythonForCommand("uv", []string{"run", "pytest"}, "python", tt.version) + installer.executor = executor + _, err := installer.Install(t.Context(), t.TempDir()) + require.NoError(t, err) + require.Equal(t, "uv", executor.commands[0].name) + require.Equal(t, []string{"run", "python", "-m", "pip"}, executor.commands[0].args[:4]) + require.Equal(t, tt.spec, executor.commands[0].args[len(executor.commands[0].args)-1]) + }) + } + _, err := NewPython("python", "git:").Install(t.Context(), t.TempDir()) + require.ErrorContains(t, err, "git ref must not be empty") +} diff --git a/internal/testdrive/tracer/python.go b/internal/testdrive/tracer/python.go index 4af48826..0adc0078 100644 --- a/internal/testdrive/tracer/python.go +++ b/internal/testdrive/tracer/python.go @@ -17,28 +17,27 @@ import ( "github.com/kballard/go-shellquote" ) -const PythonVersion = "4.15.1" - // Python installs into a private target, keeping the active interpreter and // the customer's dependencies available when pytest runs. type Python struct { Interpreter string + version string command string prefixArgs []string executor commandExecutor } -func NewPython(interpreter string) *Python { - return &Python{Interpreter: interpreter, command: interpreter, executor: &ext.DefaultCommandExecutor{}} +func NewPython(interpreter, version string) *Python { + return &Python{version: version, Interpreter: interpreter, command: interpreter, executor: &ext.DefaultCommandExecutor{}} } -func NewPythonForCommand(command string, args []string, fallback string) *Python { +func NewPythonForCommand(command string, args []string, fallback, version string) *Python { base := strings.ToLower(filepath.Base(strings.ReplaceAll(command, `\`, "/"))) if isPythonExecutable(base) { - return NewPython(command) + return NewPython(command, version) } if (base == "uv" || base == "poetry") && len(args) > 0 && args[0] == "run" { - python := NewPython(command) + python := NewPython(command, version) python.prefixArgs = []string{"run", "python"} python.Interpreter = shellquote.Join(command, "run", "python") return python @@ -48,9 +47,9 @@ func NewPythonForCommand(command string, args []string, fallback string) *Python if strings.HasSuffix(base, ".exe") { interpreter += ".exe" } - return NewPython(interpreter) + return NewPython(interpreter, version) } - return NewPython(fallback) + return NewPython(fallback, version) } func isPythonExecutable(base string) bool { @@ -67,9 +66,18 @@ func isPythonExecutable(base string) bool { } func (p *Python) Install(ctx context.Context, directory string) (string, error) { + packageName := "ddtrace" + if ref, ok := strings.CutPrefix(p.version, "git:"); ok { + if ref == "" { + return "", fmt.Errorf("tracer git ref must not be empty") + } + packageName += " @ git+https://github.com/DataDog/dd-trace-py.git@" + ref + } else if p.version != "" && p.version != "latest" { + packageName += "==" + p.version + } target := filepath.Join(directory, "python-packages") - args := append(append([]string{}, p.prefixArgs...), "-m", "pip", "install", "--disable-pip-version-check", "--target", target, "ddtrace=="+PythonVersion) - if output, err := p.executor.CombinedOutput(ctx, p.command, args, nil); err != nil { + args := append(append([]string{}, p.prefixArgs...), "-m", "pip", "install", "--disable-pip-version-check", "--target", target, packageName) + if output, err := p.executor.CombinedOutput(ctx, p.command, args, map[string]string{"DD_FAST_BUILD": "1"}); err != nil { return "", commandError("install ddtrace", output, err) } bootstrap := filepath.Join(directory, "python") @@ -77,7 +85,7 @@ func (p *Python) Install(ctx context.Context, directory string) (string, error) return "", fmt.Errorf("create Python tracer bootstrap: %w", err) } encodedTarget, _ := json.Marshal(target) - contents := "import sys\nsys.path.append(" + string(encodedTarget) + ")\n" + contents := "import sys\nsys.path.insert(0, " + string(encodedTarget) + ")\n" if err := os.WriteFile(filepath.Join(bootstrap, "sitecustomize.py"), []byte(contents), 0600); err != nil { return "", fmt.Errorf("write Python tracer bootstrap: %w", err) } From 0776b92a79afa08d52362be521c4a1e2093db324 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 14:05:05 +0200 Subject: [PATCH 15/35] Reuse Python project tracers before considering fallback installs --- internal/testdrive/multiframework_test.go | 13 ++++++ internal/testdrive/tracer/languages_test.go | 51 ++++++++++++++++----- internal/testdrive/tracer/python.go | 25 ++++++---- 3 files changed, 69 insertions(+), 20 deletions(-) diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 869f0357..0e4b7fdf 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -173,3 +173,16 @@ func TestCypressWrapperSupportsConfigFileFalseAndDefaultE2E(t *testing.T) { require.Contains(t, string(wrapper), `types.add(options.testingType)`) require.Contains(t, string(wrapper), `const originalImport = {}`) } + +func TestPythonProjectTracerPreservesImportEnvironment(t *testing.T) { + t.Setenv("PYTHONPATH", "/project/helpers") + t.Setenv("PYTEST_ADDOPTS", "-v") + drive := &Testdrive{language: "python"} + env := drive.environment("", "http://127.0.0.1:1234", "session") + if _, changed := env["PYTHONPATH"]; changed { + t.Fatal("project PYTHONPATH overridden", env) + } + if env["PYTEST_ADDOPTS"] != "-v --ddtrace" { + t.Fatal(env) + } +} diff --git a/internal/testdrive/tracer/languages_test.go b/internal/testdrive/tracer/languages_test.go index 60fef050..139b2842 100644 --- a/internal/testdrive/tracer/languages_test.go +++ b/internal/testdrive/tracer/languages_test.go @@ -16,18 +16,18 @@ import ( func TestPythonInstallUsesSelectedInterpreterAndIsolatedTarget(t *testing.T) { directory := t.TempDir() - executor := &fakeCommandExecutor{responses: []commandResponse{{}}} + executor := &fakeCommandExecutor{responses: []commandResponse{{}, {}}} installer := &Python{Interpreter: "/customer/venv/bin/python", command: "/customer/venv/bin/python", executor: executor} path, err := installer.Install(t.Context(), directory) require.NoError(t, err) - require.Equal(t, filepath.Join(directory, "python"), path) - require.Equal(t, "/customer/venv/bin/python", executor.commands[0].name) - require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", filepath.Join(directory, "python-packages"), "ddtrace"}, executor.commands[0].args) - contents, err := os.ReadFile(filepath.Join(path, "sitecustomize.py")) + require.Equal(t, filepath.Join(directory, "python"), path.Path) + require.Equal(t, "/customer/venv/bin/python", executor.commands[1].name) + require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", filepath.Join(directory, "python-packages"), "ddtrace"}, executor.commands[1].args) + contents, err := os.ReadFile(filepath.Join(path.Path, "sitecustomize.py")) require.NoError(t, err) require.Contains(t, string(contents), filepath.Join(directory, "python-packages")) - require.Contains(t, string(contents), "sys.path.insert(0, ") - executor = &fakeCommandExecutor{responses: []commandResponse{{output: []byte("pip unavailable"), err: errors.New("exit 1")}}} + require.Contains(t, string(contents), "sys.path.append(") + executor = &fakeCommandExecutor{responses: []commandResponse{{}, {output: []byte("pip unavailable"), err: errors.New("exit 1")}}} installer.executor = executor _, err = installer.Install(t.Context(), directory) require.ErrorContains(t, err, "pip unavailable") @@ -57,16 +57,45 @@ func TestPythonTracerVersions(t *testing.T) { {"git:abc1234", "ddtrace @ git+https://github.com/DataDog/dd-trace-py.git@abc1234"}, } { t.Run(tt.version, func(t *testing.T) { - executor := &fakeCommandExecutor{responses: []commandResponse{{}}} + executor := &fakeCommandExecutor{responses: []commandResponse{{}, {}}} installer := NewPythonForCommand("uv", []string{"run", "pytest"}, "python", tt.version) installer.executor = executor _, err := installer.Install(t.Context(), t.TempDir()) require.NoError(t, err) - require.Equal(t, "uv", executor.commands[0].name) - require.Equal(t, []string{"run", "python", "-m", "pip"}, executor.commands[0].args[:4]) - require.Equal(t, tt.spec, executor.commands[0].args[len(executor.commands[0].args)-1]) + require.Equal(t, "uv", executor.commands[1].name) + require.Equal(t, []string{"run", "python", "-m", "pip"}, executor.commands[1].args[:4]) + require.Equal(t, tt.spec, executor.commands[1].args[len(executor.commands[1].args)-1]) }) } _, err := NewPython("python", "git:").Install(t.Context(), t.TempDir()) require.ErrorContains(t, err, "git ref must not be empty") } + +func TestPythonReusesProjectTracer(t *testing.T) { + for _, version := range []string{"latest", "4.15.1", "git:abc1234"} { + t.Run(version, func(t *testing.T) { + executor := &fakeCommandExecutor{responses: []commandResponse{{output: []byte("3.0.0\n")}}} + installer := NewPythonForCommand("uv", []string{"run", "pytest"}, "python", version) + installer.executor = executor + directory := t.TempDir() + result, err := installer.Install(t.Context(), directory) + require.NoError(t, err) + require.Equal(t, Installation{Project: true}, result) + require.Len(t, executor.commands, 1) + require.Equal(t, "uv", executor.commands[0].name) + require.Equal(t, []string{"run", "python", "-c"}, executor.commands[0].args[:3]) + entries, err := os.ReadDir(directory) + require.NoError(t, err) + require.Empty(t, entries) + }) + } +} + +func TestPythonProbeFailureDoesNotInstall(t *testing.T) { + executor := &fakeCommandExecutor{responses: []commandResponse{{err: errors.New("interpreter unavailable")}}} + installer := NewPython("python", "latest") + installer.executor = executor + _, err := installer.Install(t.Context(), t.TempDir()) + require.ErrorContains(t, err, "interpreter unavailable") + require.Len(t, executor.commands, 1) +} diff --git a/internal/testdrive/tracer/python.go b/internal/testdrive/tracer/python.go index 0adc0078..c1fcec19 100644 --- a/internal/testdrive/tracer/python.go +++ b/internal/testdrive/tracer/python.go @@ -14,11 +14,11 @@ import ( "strings" "github.com/DataDog/ddtest/internal/ext" + "github.com/DataDog/ddtest/internal/platform" "github.com/kballard/go-shellquote" ) -// Python installs into a private target, keeping the active interpreter and -// the customer's dependencies available when pytest runs. +// Python reuses the active interpreter's tracer, installing into a private target only when absent. type Python struct { Interpreter string version string @@ -65,29 +65,36 @@ func isPythonExecutable(base string) bool { return true } -func (p *Python) Install(ctx context.Context, directory string) (string, error) { +func (p *Python) Install(ctx context.Context, directory string) (Installation, error) { packageName := "ddtrace" if ref, ok := strings.CutPrefix(p.version, "git:"); ok { if ref == "" { - return "", fmt.Errorf("tracer git ref must not be empty") + return Installation{}, fmt.Errorf("tracer git ref must not be empty") } packageName += " @ git+https://github.com/DataDog/dd-trace-py.git@" + ref } else if p.version != "" && p.version != "latest" { packageName += "==" + p.version } + version, err := platform.DetectPythonTracer(ctx, p.executor, p.command, p.prefixArgs) + if err != nil { + return Installation{}, err + } + if version != "" { + return Installation{Project: true}, nil + } target := filepath.Join(directory, "python-packages") args := append(append([]string{}, p.prefixArgs...), "-m", "pip", "install", "--disable-pip-version-check", "--target", target, packageName) if output, err := p.executor.CombinedOutput(ctx, p.command, args, map[string]string{"DD_FAST_BUILD": "1"}); err != nil { - return "", commandError("install ddtrace", output, err) + return Installation{}, commandError("install ddtrace", output, err) } bootstrap := filepath.Join(directory, "python") if err := os.MkdirAll(bootstrap, 0755); err != nil { - return "", fmt.Errorf("create Python tracer bootstrap: %w", err) + return Installation{}, fmt.Errorf("create Python tracer bootstrap: %w", err) } encodedTarget, _ := json.Marshal(target) - contents := "import sys\nsys.path.insert(0, " + string(encodedTarget) + ")\n" + contents := "import sys\nsys.path.append(" + string(encodedTarget) + ")\n" if err := os.WriteFile(filepath.Join(bootstrap, "sitecustomize.py"), []byte(contents), 0600); err != nil { - return "", fmt.Errorf("write Python tracer bootstrap: %w", err) + return Installation{}, fmt.Errorf("write Python tracer bootstrap: %w", err) } - return bootstrap, nil + return Installation{Path: bootstrap}, nil } From 115d2aee9e26c7555b95a50a293f96afea14c3db Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 15:31:04 +0200 Subject: [PATCH 16/35] Use Platform-owned tracer preparation in 144 --- internal/testdrive/multiframework_test.go | 1 + internal/testdrive/python.go | 19 ++++ internal/testdrive/testdrive.go | 4 +- internal/testdrive/tracer/languages_test.go | 101 -------------------- internal/testdrive/tracer/python.go | 100 ------------------- 5 files changed, 23 insertions(+), 202 deletions(-) create mode 100644 internal/testdrive/python.go delete mode 100644 internal/testdrive/tracer/languages_test.go delete mode 100644 internal/testdrive/tracer/python.go diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 0e4b7fdf..9e484f32 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -42,6 +42,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { run.nodeVersion = func() string { return "v20.0.0" } installer := &fakeTracer{preloadPath: filepath.Join(root, "isolated")} run.platform = installer + run.projectTracer = "" executor := &fakeTestdriveExecutor{} run.executor = executor run.startIntake = func(string) (localIntake, error) { diff --git a/internal/testdrive/python.go b/internal/testdrive/python.go new file mode 100644 index 00000000..785fedd5 --- /dev/null +++ b/internal/testdrive/python.go @@ -0,0 +1,19 @@ +package testdrive + +import ( + "os" + "strings" +) + +func pythonEnvironment(path string) map[string]string { + env := map[string]string{} + if path != "" { + env["PYTHONPATH"] = path + if existing := os.Getenv("PYTHONPATH"); existing != "" { + env["PYTHONPATH"] += string(os.PathListSeparator) + existing + } + } + env["PYTEST_ADDOPTS"] = strings.TrimSpace(os.Getenv("PYTEST_ADDOPTS") + " --ddtrace") + + return env +} diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index cd6728d0..14583496 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -80,7 +80,7 @@ func Prepare(version string) (*Testdrive, error) { } language := detectedPlatform.Name() switch runner.Name() { - case "jest", "mocha", "vitest", "playwright", "cucumber", "cypress": + case "jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest": default: return nil, fmt.Errorf("testdrive does not yet support %s", runner.Name()) } @@ -424,6 +424,8 @@ func (t *Testdrive) environment(path, intakeURL, sessionID string) map[string]st switch t.language { case "javascript": maps.Copy(env, t.javascriptEnvironment(path)) + case "python": + maps.Copy(env, pythonEnvironment(path)) } return env } diff --git a/internal/testdrive/tracer/languages_test.go b/internal/testdrive/tracer/languages_test.go deleted file mode 100644 index 139b2842..00000000 --- a/internal/testdrive/tracer/languages_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026 Datadog, Inc. - -package tracer - -import ( - "errors" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestPythonInstallUsesSelectedInterpreterAndIsolatedTarget(t *testing.T) { - directory := t.TempDir() - executor := &fakeCommandExecutor{responses: []commandResponse{{}, {}}} - installer := &Python{Interpreter: "/customer/venv/bin/python", command: "/customer/venv/bin/python", executor: executor} - path, err := installer.Install(t.Context(), directory) - require.NoError(t, err) - require.Equal(t, filepath.Join(directory, "python"), path.Path) - require.Equal(t, "/customer/venv/bin/python", executor.commands[1].name) - require.Equal(t, []string{"-m", "pip", "install", "--disable-pip-version-check", "--target", filepath.Join(directory, "python-packages"), "ddtrace"}, executor.commands[1].args) - contents, err := os.ReadFile(filepath.Join(path.Path, "sitecustomize.py")) - require.NoError(t, err) - require.Contains(t, string(contents), filepath.Join(directory, "python-packages")) - require.Contains(t, string(contents), "sys.path.append(") - executor = &fakeCommandExecutor{responses: []commandResponse{{}, {output: []byte("pip unavailable"), err: errors.New("exit 1")}}} - installer.executor = executor - _, err = installer.Install(t.Context(), directory) - require.ErrorContains(t, err, "pip unavailable") -} -func TestNewPythonForCommandUsesRunnerInterpreter(t *testing.T) { - for _, test := range []struct { - command string - args []string - wantCommand string - wantPrefix []string - }{ - {command: "python3.12", args: []string{"-m", "pytest"}, wantCommand: "python3.12"}, - {command: ".venv/bin/pytest", wantCommand: filepath.Join(".venv", "bin", "python")}, - {command: "uv", args: []string{"run", "pytest"}, wantCommand: "uv", wantPrefix: []string{"run", "python"}}, - {command: "poetry", args: []string{"run", "pytest"}, wantCommand: "poetry", wantPrefix: []string{"run", "python"}}, - } { - installer := NewPythonForCommand(test.command, test.args, "python", "latest") - require.Equal(t, test.wantCommand, installer.command) - require.Equal(t, test.wantPrefix, installer.prefixArgs) - } -} - -func TestPythonTracerVersions(t *testing.T) { - for _, tt := range []struct{ version, spec string }{ - {"", "ddtrace"}, {"latest", "ddtrace"}, {"4.15.1", "ddtrace==4.15.1"}, - {"4.16.0rc1", "ddtrace==4.16.0rc1"}, - {"git:abc1234", "ddtrace @ git+https://github.com/DataDog/dd-trace-py.git@abc1234"}, - } { - t.Run(tt.version, func(t *testing.T) { - executor := &fakeCommandExecutor{responses: []commandResponse{{}, {}}} - installer := NewPythonForCommand("uv", []string{"run", "pytest"}, "python", tt.version) - installer.executor = executor - _, err := installer.Install(t.Context(), t.TempDir()) - require.NoError(t, err) - require.Equal(t, "uv", executor.commands[1].name) - require.Equal(t, []string{"run", "python", "-m", "pip"}, executor.commands[1].args[:4]) - require.Equal(t, tt.spec, executor.commands[1].args[len(executor.commands[1].args)-1]) - }) - } - _, err := NewPython("python", "git:").Install(t.Context(), t.TempDir()) - require.ErrorContains(t, err, "git ref must not be empty") -} - -func TestPythonReusesProjectTracer(t *testing.T) { - for _, version := range []string{"latest", "4.15.1", "git:abc1234"} { - t.Run(version, func(t *testing.T) { - executor := &fakeCommandExecutor{responses: []commandResponse{{output: []byte("3.0.0\n")}}} - installer := NewPythonForCommand("uv", []string{"run", "pytest"}, "python", version) - installer.executor = executor - directory := t.TempDir() - result, err := installer.Install(t.Context(), directory) - require.NoError(t, err) - require.Equal(t, Installation{Project: true}, result) - require.Len(t, executor.commands, 1) - require.Equal(t, "uv", executor.commands[0].name) - require.Equal(t, []string{"run", "python", "-c"}, executor.commands[0].args[:3]) - entries, err := os.ReadDir(directory) - require.NoError(t, err) - require.Empty(t, entries) - }) - } -} - -func TestPythonProbeFailureDoesNotInstall(t *testing.T) { - executor := &fakeCommandExecutor{responses: []commandResponse{{err: errors.New("interpreter unavailable")}}} - installer := NewPython("python", "latest") - installer.executor = executor - _, err := installer.Install(t.Context(), t.TempDir()) - require.ErrorContains(t, err, "interpreter unavailable") - require.Len(t, executor.commands, 1) -} diff --git a/internal/testdrive/tracer/python.go b/internal/testdrive/tracer/python.go deleted file mode 100644 index c1fcec19..00000000 --- a/internal/testdrive/tracer/python.go +++ /dev/null @@ -1,100 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026 Datadog, Inc. - -package tracer - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/DataDog/ddtest/internal/ext" - "github.com/DataDog/ddtest/internal/platform" - "github.com/kballard/go-shellquote" -) - -// Python reuses the active interpreter's tracer, installing into a private target only when absent. -type Python struct { - Interpreter string - version string - command string - prefixArgs []string - executor commandExecutor -} - -func NewPython(interpreter, version string) *Python { - return &Python{version: version, Interpreter: interpreter, command: interpreter, executor: &ext.DefaultCommandExecutor{}} -} - -func NewPythonForCommand(command string, args []string, fallback, version string) *Python { - base := strings.ToLower(filepath.Base(strings.ReplaceAll(command, `\`, "/"))) - if isPythonExecutable(base) { - return NewPython(command, version) - } - if (base == "uv" || base == "poetry") && len(args) > 0 && args[0] == "run" { - python := NewPython(command, version) - python.prefixArgs = []string{"run", "python"} - python.Interpreter = shellquote.Join(command, "run", "python") - return python - } - if strings.HasPrefix(base, "pytest") && filepath.Dir(command) != "." { - interpreter := filepath.Join(filepath.Dir(command), "python") - if strings.HasSuffix(base, ".exe") { - interpreter += ".exe" - } - return NewPython(interpreter, version) - } - return NewPython(fallback, version) -} - -func isPythonExecutable(base string) bool { - base = strings.TrimSuffix(base, ".exe") - if !strings.HasPrefix(base, "python") { - return false - } - for _, character := range strings.TrimPrefix(base, "python") { - if (character < '0' || character > '9') && character != '.' { - return false - } - } - return true -} - -func (p *Python) Install(ctx context.Context, directory string) (Installation, error) { - packageName := "ddtrace" - if ref, ok := strings.CutPrefix(p.version, "git:"); ok { - if ref == "" { - return Installation{}, fmt.Errorf("tracer git ref must not be empty") - } - packageName += " @ git+https://github.com/DataDog/dd-trace-py.git@" + ref - } else if p.version != "" && p.version != "latest" { - packageName += "==" + p.version - } - version, err := platform.DetectPythonTracer(ctx, p.executor, p.command, p.prefixArgs) - if err != nil { - return Installation{}, err - } - if version != "" { - return Installation{Project: true}, nil - } - target := filepath.Join(directory, "python-packages") - args := append(append([]string{}, p.prefixArgs...), "-m", "pip", "install", "--disable-pip-version-check", "--target", target, packageName) - if output, err := p.executor.CombinedOutput(ctx, p.command, args, map[string]string{"DD_FAST_BUILD": "1"}); err != nil { - return Installation{}, commandError("install ddtrace", output, err) - } - bootstrap := filepath.Join(directory, "python") - if err := os.MkdirAll(bootstrap, 0755); err != nil { - return Installation{}, fmt.Errorf("create Python tracer bootstrap: %w", err) - } - encodedTarget, _ := json.Marshal(target) - contents := "import sys\nsys.path.append(" + string(encodedTarget) + ")\n" - if err := os.WriteFile(filepath.Join(bootstrap, "sitecustomize.py"), []byte(contents), 0600); err != nil { - return Installation{}, fmt.Errorf("write Python tracer bootstrap: %w", err) - } - return Installation{Path: bootstrap}, nil -} From f76899ddb10d5dda40e9f5db62bc4cf6c714ba6b Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:15:57 +0200 Subject: [PATCH 17/35] Support isolated RSpec and Minitest testdrives --- .../testdrive/frameworks_integration_test.go | 2 + internal/testdrive/multiframework_test.go | 7 +- internal/testdrive/tracer/ruby.go | 88 +++++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 internal/testdrive/tracer/ruby.go diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index bedaf9f3..05b0f6f8 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -39,6 +39,8 @@ func TestPublicFrameworkTestdrives(t *testing.T) { {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, {"cypress", `{"scripts":{"test":"cypress run"},"devDependencies":{"cypress":"15.1.0"}}`, "npm test", map[string]string{"cypress.config.js": `module.exports={e2e:{supportFile:false,setupNodeEvents(on,config){on('task',{answer:()=>42});on('after:run',()=>{require('node:fs').writeFileSync('original-hook.txt','ran');});return config;}}};`, "cypress/e2e/one.cy.js": `it('preserves hooks', () => { cy.task('answer').should('equal',42); });`}}, {"pytest", "", "python -m pytest", map[string]string{"requirements.txt": "pytest==8.4.2\n", "pytest.ini": "[pytest]\n", "test_one.py": "def test_adds():\n assert 1 + 1 == 2\n"}}, + {"rspec", "", "bundle exec rspec", map[string]string{"Gemfile": "source 'https://rubygems.org'\ngem 'rspec', '3.13.1'\n", "spec/one_spec.rb": "RSpec.describe 'addition' do\n it('adds') { expect(1 + 1).to eq(2) }\nend\n"}}, + {"minitest", "", "bundle exec rake test", map[string]string{"Gemfile": "source 'https://rubygems.org'\ngem 'rake', '13.3.0'\ngem 'minitest', '5.25.5'\n", "Rakefile": "require 'rake/testtask'\nRake::TestTask.new(:test) { |t| t.pattern = 'test/**/*_test.rb' }\n", "test/one_test.rb": "require 'minitest/autorun'\nclass AdditionTest < Minitest::Test\n def test_adds\n assert_equal 2, 1 + 1\n end\nend\n"}}, } for _, fixture := range fixtures { t.Run(fixture.name, func(t *testing.T) { diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 9e484f32..642de476 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -10,6 +10,7 @@ import ( "github.com/DataDog/ddtest/internal/settings" "os" "path/filepath" + "strings" "testing" "github.com/DataDog/ddtest/internal/testdrive/intake" @@ -17,7 +18,7 @@ import ( ) func TestPrepareAllSupportedFrameworks(t *testing.T) { - for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest"} { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest", "rspec", "minitest"} { t.Run(name, func(t *testing.T) { root := t.TempDir() switch name { @@ -125,7 +126,9 @@ func TestLanguageEnvironmentsPreserveCustomerOptions(t *testing.T) { python := (&Testdrive{language: "python"}).environment("/session/python", "http://127.0.0.1:1234", "session") require.Equal(t, "/session/python"+string(os.PathListSeparator)+"/customer/modules", python["PYTHONPATH"]) require.Equal(t, "-q --ddtrace", python["PYTEST_ADDOPTS"]) - + ruby := (&Testdrive{language: "ruby"}).environment("/session/Gemfile", "http://127.0.0.1:1234", "session") + require.True(t, strings.HasPrefix(ruby["RUBYOPT"], "-W0 ")) + require.Equal(t, "/session/gems", ruby["BUNDLE_PATH"]) } func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { diff --git a/internal/testdrive/tracer/ruby.go b/internal/testdrive/tracer/ruby.go new file mode 100644 index 00000000..f199d812 --- /dev/null +++ b/internal/testdrive/tracer/ruby.go @@ -0,0 +1,88 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package tracer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/DataDog/ddtest/internal/ext" +) + +const RubyVersion = "1.39.0" + +// Ruby resolves an overlay bundle inside the session. Bundler sees the project's +// original Gemfile (including relative gemspecs), but never writes its lockfile. +type Ruby struct { + root string + executor commandExecutor +} + +func NewRuby(root string) *Ruby { return &Ruby{root: root, executor: &ext.DefaultCommandExecutor{}} } + +func (r *Ruby) Install(ctx context.Context, directory string) (string, error) { + if strings.Contains(directory, " ") { + return "", fmt.Errorf("the pinned Ruby tracer's native extensions cannot build in paths containing spaces; run testdrive from a checkout without spaces") + } + gemfile := filepath.Join(directory, "Gemfile") + path := strings.ReplaceAll(strings.ReplaceAll(filepath.Join(r.root, "Gemfile"), `\`, `\\`), "'", `\'`) + contents := "source 'https://rubygems.org'\neval_gemfile '" + path + "'\n" + + "dependencies.reject! { |dependency| dependency.name == 'datadog-ci' }\n" + + "gem 'datadog-ci', '" + RubyVersion + "'\n" + if err := os.WriteFile(gemfile, []byte(contents), 0600); err != nil { + return "", fmt.Errorf("write isolated Gemfile: %w", err) + } + if err := copyRubyLockfile(r.root, directory); err != nil { + return "", err + } + env := RubyEnvironment(gemfile) + if output, err := r.executor.CombinedOutput(ctx, "bundle", []string{"install"}, env); err != nil { + return "", commandError("install isolated Ruby bundle", output, err) + } + return gemfile, nil +} + +// Preserve the customer's resolved versions while adding the tracer. PATH +// sources in a lockfile are relative to its Gemfile, so relocate those sources +// when copying it into the session. The original remains untouched. +func copyRubyLockfile(root, directory string) error { + contents, err := os.ReadFile(filepath.Join(root, "Gemfile.lock")) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("read project lockfile: %w", err) + } + lines := strings.Split(string(contents), "\n") + inPath := false + for i, line := range lines { + if line != "" && !strings.HasPrefix(line, " ") { + inPath = line == "PATH" + } + if inPath && strings.HasPrefix(line, " remote: ") { + path := strings.TrimPrefix(line, " remote: ") + if !filepath.IsAbs(path) { + lines[i] = " remote: " + filepath.Join(root, path) + } + } + } + if err := os.WriteFile(filepath.Join(directory, "Gemfile.lock"), []byte(strings.Join(lines, "\n")), 0600); err != nil { + return fmt.Errorf("copy project lockfile: %w", err) + } + return nil +} + +func RubyEnvironment(gemfile string) map[string]string { + return map[string]string{ + "BUNDLE_GEMFILE": gemfile, + "BUNDLE_PATH": filepath.Join(filepath.Dir(gemfile), "gems"), + "BUNDLE_APP_CONFIG": filepath.Join(filepath.Dir(gemfile), "bundle-config"), + "BUNDLE_FROZEN": "false", "BUNDLE_DEPLOYMENT": "false", + } +} From 263b6a11d9278c938287d175f9e22422ec6d7f9f Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 09:15:57 +0200 Subject: [PATCH 18/35] Preserve isolated Ruby project settings --- internal/testdrive/command_test.go | 11 +++++++++++ internal/testdrive/tracer/ruby.go | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/internal/testdrive/command_test.go b/internal/testdrive/command_test.go index 7a82ed3e..7a0958db 100644 --- a/internal/testdrive/command_test.go +++ b/internal/testdrive/command_test.go @@ -2,6 +2,7 @@ package testdrive import ( "os" + "path/filepath" "testing" "github.com/DataDog/ddtest/internal/framework" @@ -43,3 +44,13 @@ func TestTestdrivePreservesExplicitCommandArguments(t *testing.T) { require.Equal(t, "npm", command) require.Equal(t, []string{"run", "smoke", "--", "--config", "config with spaces.js"}, args) } + +func TestTestdriveMinitestRequiresExecutableRailsBinstub(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "bin", "rails"), []byte("#!/bin/sh\n"), 0644)) + command, args, err := TestdriveCommand(root, NewMinitest()) + require.NoError(t, err) + require.Equal(t, "bundle", command) + require.Equal(t, []string{"exec", "rake", "test"}, args) +} diff --git a/internal/testdrive/tracer/ruby.go b/internal/testdrive/tracer/ruby.go index f199d812..6e4659ab 100644 --- a/internal/testdrive/tracer/ruby.go +++ b/internal/testdrive/tracer/ruby.go @@ -41,6 +41,9 @@ func (r *Ruby) Install(ctx context.Context, directory string) (string, error) { if err := copyRubyLockfile(r.root, directory); err != nil { return "", err } + if err := copyRubyBundleConfig(r.root, directory); err != nil { + return "", err + } env := RubyEnvironment(gemfile) if output, err := r.executor.CombinedOutput(ctx, "bundle", []string{"install"}, env); err != nil { return "", commandError("install isolated Ruby bundle", output, err) @@ -48,6 +51,24 @@ func (r *Ruby) Install(ctx context.Context, directory string) (string, error) { return gemfile, nil } +func copyRubyBundleConfig(root, directory string) error { + contents, err := os.ReadFile(filepath.Join(root, ".bundle", "config")) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("read project Bundler config: %w", err) + } + configDirectory := filepath.Join(directory, "bundle-config") + if err := os.MkdirAll(configDirectory, 0700); err != nil { + return fmt.Errorf("create isolated Bundler config: %w", err) + } + if err := os.WriteFile(filepath.Join(configDirectory, "config"), contents, 0600); err != nil { + return fmt.Errorf("copy project Bundler config: %w", err) + } + return nil +} + // Preserve the customer's resolved versions while adding the tracer. PATH // sources in a lockfile are relative to its Gemfile, so relocate those sources // when copying it into the session. The original remains untouched. @@ -84,5 +105,6 @@ func RubyEnvironment(gemfile string) map[string]string { "BUNDLE_PATH": filepath.Join(filepath.Dir(gemfile), "gems"), "BUNDLE_APP_CONFIG": filepath.Join(filepath.Dir(gemfile), "bundle-config"), "BUNDLE_FROZEN": "false", "BUNDLE_DEPLOYMENT": "false", + "BUNDLE_WITH": "", "BUNDLE_WITHOUT": "", "BUNDLE_ONLY": "", } } From 846b1e7cfbf9324322113af5f686c9d9161b648f Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 13:38:30 +0200 Subject: [PATCH 19/35] Select latest Ruby tracer or a release and Git revision --- internal/testdrive/tracer/ruby.go | 32 +++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/internal/testdrive/tracer/ruby.go b/internal/testdrive/tracer/ruby.go index 6e4659ab..f6082e72 100644 --- a/internal/testdrive/tracer/ruby.go +++ b/internal/testdrive/tracer/ruby.go @@ -15,26 +15,37 @@ import ( "github.com/DataDog/ddtest/internal/ext" ) -const RubyVersion = "1.39.0" - // Ruby resolves an overlay bundle inside the session. Bundler sees the project's // original Gemfile (including relative gemspecs), but never writes its lockfile. type Ruby struct { root string + version string executor commandExecutor } -func NewRuby(root string) *Ruby { return &Ruby{root: root, executor: &ext.DefaultCommandExecutor{}} } +func NewRuby(root, version string) *Ruby { + return &Ruby{root: root, version: version, executor: &ext.DefaultCommandExecutor{}} +} func (r *Ruby) Install(ctx context.Context, directory string) (string, error) { if strings.Contains(directory, " ") { - return "", fmt.Errorf("the pinned Ruby tracer's native extensions cannot build in paths containing spaces; run testdrive from a checkout without spaces") + return "", fmt.Errorf("the Ruby tracer's native extensions cannot build in paths containing spaces; run testdrive from a checkout without spaces") + } + selection := "" + version := strings.NewReplacer(`\`, `\\`, "'", `\'`).Replace(r.version) + if ref, ok := strings.CutPrefix(version, "git:"); ok { + if ref == "" { + return "", fmt.Errorf("tracer git ref must not be empty") + } + selection = ", git: 'https://github.com/DataDog/datadog-ci-rb.git', ref: '" + ref + "'" + } else if version != "" && version != "latest" { + selection = ", '" + version + "'" } gemfile := filepath.Join(directory, "Gemfile") path := strings.ReplaceAll(strings.ReplaceAll(filepath.Join(r.root, "Gemfile"), `\`, `\\`), "'", `\'`) contents := "source 'https://rubygems.org'\neval_gemfile '" + path + "'\n" + "dependencies.reject! { |dependency| dependency.name == 'datadog-ci' }\n" + - "gem 'datadog-ci', '" + RubyVersion + "'\n" + "gem 'datadog-ci'" + selection + "\n" if err := os.WriteFile(gemfile, []byte(contents), 0600); err != nil { return "", fmt.Errorf("write isolated Gemfile: %w", err) } @@ -44,8 +55,17 @@ func (r *Ruby) Install(ctx context.Context, directory string) (string, error) { if err := copyRubyBundleConfig(r.root, directory); err != nil { return "", err } + args := []string{"install"} + lock, err := os.ReadFile(filepath.Join(directory, "Gemfile.lock")) + if err != nil && !os.IsNotExist(err) { + return "", fmt.Errorf("read isolated lockfile: %w", err) + } + // Refresh a copied tracer pin while preserving other locked dependencies. + if strings.Contains(string(lock), "\n datadog-ci (") { + args = []string{"update", "datadog-ci", "--conservative"} + } env := RubyEnvironment(gemfile) - if output, err := r.executor.CombinedOutput(ctx, "bundle", []string{"install"}, env); err != nil { + if output, err := r.executor.CombinedOutput(ctx, "bundle", args, env); err != nil { return "", commandError("install isolated Ruby bundle", output, err) } return gemfile, nil From 3afdd529cd4f7bc3476e9e8a2f492c69a84d3773 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 14:09:11 +0200 Subject: [PATCH 20/35] Reuse the project Ruby tracer without replacing its bundle --- internal/testdrive/multiframework_test.go | 13 ++++++ internal/testdrive/tracer/ruby.go | 50 ++++++++++++----------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 642de476..d4a47bd0 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -190,3 +190,16 @@ func TestPythonProjectTracerPreservesImportEnvironment(t *testing.T) { t.Fatal(env) } } + +func TestRubyProjectTracerPreservesBundleEnvironment(t *testing.T) { + drive := &Testdrive{language: "ruby"} + env := drive.environment("", "http://127.0.0.1:1234", "session") + for _, key := range []string{"BUNDLE_GEMFILE", "BUNDLE_PATH", "BUNDLE_APP_CONFIG", "BUNDLE_FROZEN", "BUNDLE_WITHOUT"} { + if _, changed := env[key]; changed { + t.Fatal("project Bundler setting overridden", key) + } + } + if !strings.Contains(env["RUBYOPT"], "-rdatadog/ci/auto_instrument") { + t.Fatal(env) + } +} diff --git a/internal/testdrive/tracer/ruby.go b/internal/testdrive/tracer/ruby.go index f6082e72..124e422e 100644 --- a/internal/testdrive/tracer/ruby.go +++ b/internal/testdrive/tracer/ruby.go @@ -13,10 +13,10 @@ import ( "strings" "github.com/DataDog/ddtest/internal/ext" + "github.com/DataDog/ddtest/internal/platform" ) -// Ruby resolves an overlay bundle inside the session. Bundler sees the project's -// original Gemfile (including relative gemspecs), but never writes its lockfile. +// Ruby reuses the project tracer, adding an isolated overlay bundle only when absent. type Ruby struct { root string version string @@ -27,48 +27,49 @@ func NewRuby(root, version string) *Ruby { return &Ruby{root: root, version: version, executor: &ext.DefaultCommandExecutor{}} } -func (r *Ruby) Install(ctx context.Context, directory string) (string, error) { - if strings.Contains(directory, " ") { - return "", fmt.Errorf("the Ruby tracer's native extensions cannot build in paths containing spaces; run testdrive from a checkout without spaces") - } +func (r *Ruby) Install(ctx context.Context, directory string) (Installation, error) { selection := "" version := strings.NewReplacer(`\`, `\\`, "'", `\'`).Replace(r.version) if ref, ok := strings.CutPrefix(version, "git:"); ok { if ref == "" { - return "", fmt.Errorf("tracer git ref must not be empty") + return Installation{}, fmt.Errorf("tracer git ref must not be empty") } selection = ", git: 'https://github.com/DataDog/datadog-ci-rb.git', ref: '" + ref + "'" } else if version != "" && version != "latest" { selection = ", '" + version + "'" } + projectGemfile := os.Getenv("BUNDLE_GEMFILE") + if projectGemfile == "" { + projectGemfile = filepath.Join(r.root, "Gemfile") + } + project, err := platform.DetectRubyTracer(ctx, r.executor, map[string]string{"BUNDLE_GEMFILE": projectGemfile, "RUBYOPT": ""}) + if err != nil { + return Installation{}, err + } + if project != "" { + return Installation{Project: true}, nil + } + if strings.Contains(directory, " ") { + return Installation{}, fmt.Errorf("the Ruby tracer's native extensions cannot build in paths containing spaces; run testdrive from a checkout without spaces") + } gemfile := filepath.Join(directory, "Gemfile") path := strings.ReplaceAll(strings.ReplaceAll(filepath.Join(r.root, "Gemfile"), `\`, `\\`), "'", `\'`) contents := "source 'https://rubygems.org'\neval_gemfile '" + path + "'\n" + - "dependencies.reject! { |dependency| dependency.name == 'datadog-ci' }\n" + "gem 'datadog-ci'" + selection + "\n" if err := os.WriteFile(gemfile, []byte(contents), 0600); err != nil { - return "", fmt.Errorf("write isolated Gemfile: %w", err) + return Installation{}, fmt.Errorf("write isolated Gemfile: %w", err) } if err := copyRubyLockfile(r.root, directory); err != nil { - return "", err + return Installation{}, err } if err := copyRubyBundleConfig(r.root, directory); err != nil { - return "", err - } - args := []string{"install"} - lock, err := os.ReadFile(filepath.Join(directory, "Gemfile.lock")) - if err != nil && !os.IsNotExist(err) { - return "", fmt.Errorf("read isolated lockfile: %w", err) - } - // Refresh a copied tracer pin while preserving other locked dependencies. - if strings.Contains(string(lock), "\n datadog-ci (") { - args = []string{"update", "datadog-ci", "--conservative"} + return Installation{}, err } env := RubyEnvironment(gemfile) - if output, err := r.executor.CombinedOutput(ctx, "bundle", args, env); err != nil { - return "", commandError("install isolated Ruby bundle", output, err) + if output, err := r.executor.CombinedOutput(ctx, "bundle", []string{"install"}, env); err != nil { + return Installation{}, commandError("install isolated Ruby bundle", output, err) } - return gemfile, nil + return Installation{Path: gemfile}, nil } func copyRubyBundleConfig(root, directory string) error { @@ -120,6 +121,9 @@ func copyRubyLockfile(root, directory string) error { } func RubyEnvironment(gemfile string) map[string]string { + if gemfile == "" { + return nil + } return map[string]string{ "BUNDLE_GEMFILE": gemfile, "BUNDLE_PATH": filepath.Join(filepath.Dir(gemfile), "gems"), From 567214d30908d9621aebcde0a02e0b71daf81e9f Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 15:31:19 +0200 Subject: [PATCH 21/35] Use Platform-owned tracer preparation in 145 --- internal/testdrive/multiframework_test.go | 2 +- internal/testdrive/tracer/ruby.go | 134 ---------------------- 2 files changed, 1 insertion(+), 135 deletions(-) delete mode 100644 internal/testdrive/tracer/ruby.go diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index d4a47bd0..602ccb43 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -128,7 +128,7 @@ func TestLanguageEnvironmentsPreserveCustomerOptions(t *testing.T) { require.Equal(t, "-q --ddtrace", python["PYTEST_ADDOPTS"]) ruby := (&Testdrive{language: "ruby"}).environment("/session/Gemfile", "http://127.0.0.1:1234", "session") require.True(t, strings.HasPrefix(ruby["RUBYOPT"], "-W0 ")) - require.Equal(t, "/session/gems", ruby["BUNDLE_PATH"]) + require.NotContains(t, ruby, "BUNDLE_PATH") // Supplied by the platform installation result. } func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { diff --git a/internal/testdrive/tracer/ruby.go b/internal/testdrive/tracer/ruby.go deleted file mode 100644 index 124e422e..00000000 --- a/internal/testdrive/tracer/ruby.go +++ /dev/null @@ -1,134 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2026 Datadog, Inc. - -package tracer - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/DataDog/ddtest/internal/ext" - "github.com/DataDog/ddtest/internal/platform" -) - -// Ruby reuses the project tracer, adding an isolated overlay bundle only when absent. -type Ruby struct { - root string - version string - executor commandExecutor -} - -func NewRuby(root, version string) *Ruby { - return &Ruby{root: root, version: version, executor: &ext.DefaultCommandExecutor{}} -} - -func (r *Ruby) Install(ctx context.Context, directory string) (Installation, error) { - selection := "" - version := strings.NewReplacer(`\`, `\\`, "'", `\'`).Replace(r.version) - if ref, ok := strings.CutPrefix(version, "git:"); ok { - if ref == "" { - return Installation{}, fmt.Errorf("tracer git ref must not be empty") - } - selection = ", git: 'https://github.com/DataDog/datadog-ci-rb.git', ref: '" + ref + "'" - } else if version != "" && version != "latest" { - selection = ", '" + version + "'" - } - projectGemfile := os.Getenv("BUNDLE_GEMFILE") - if projectGemfile == "" { - projectGemfile = filepath.Join(r.root, "Gemfile") - } - project, err := platform.DetectRubyTracer(ctx, r.executor, map[string]string{"BUNDLE_GEMFILE": projectGemfile, "RUBYOPT": ""}) - if err != nil { - return Installation{}, err - } - if project != "" { - return Installation{Project: true}, nil - } - if strings.Contains(directory, " ") { - return Installation{}, fmt.Errorf("the Ruby tracer's native extensions cannot build in paths containing spaces; run testdrive from a checkout without spaces") - } - gemfile := filepath.Join(directory, "Gemfile") - path := strings.ReplaceAll(strings.ReplaceAll(filepath.Join(r.root, "Gemfile"), `\`, `\\`), "'", `\'`) - contents := "source 'https://rubygems.org'\neval_gemfile '" + path + "'\n" + - "gem 'datadog-ci'" + selection + "\n" - if err := os.WriteFile(gemfile, []byte(contents), 0600); err != nil { - return Installation{}, fmt.Errorf("write isolated Gemfile: %w", err) - } - if err := copyRubyLockfile(r.root, directory); err != nil { - return Installation{}, err - } - if err := copyRubyBundleConfig(r.root, directory); err != nil { - return Installation{}, err - } - env := RubyEnvironment(gemfile) - if output, err := r.executor.CombinedOutput(ctx, "bundle", []string{"install"}, env); err != nil { - return Installation{}, commandError("install isolated Ruby bundle", output, err) - } - return Installation{Path: gemfile}, nil -} - -func copyRubyBundleConfig(root, directory string) error { - contents, err := os.ReadFile(filepath.Join(root, ".bundle", "config")) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return fmt.Errorf("read project Bundler config: %w", err) - } - configDirectory := filepath.Join(directory, "bundle-config") - if err := os.MkdirAll(configDirectory, 0700); err != nil { - return fmt.Errorf("create isolated Bundler config: %w", err) - } - if err := os.WriteFile(filepath.Join(configDirectory, "config"), contents, 0600); err != nil { - return fmt.Errorf("copy project Bundler config: %w", err) - } - return nil -} - -// Preserve the customer's resolved versions while adding the tracer. PATH -// sources in a lockfile are relative to its Gemfile, so relocate those sources -// when copying it into the session. The original remains untouched. -func copyRubyLockfile(root, directory string) error { - contents, err := os.ReadFile(filepath.Join(root, "Gemfile.lock")) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return fmt.Errorf("read project lockfile: %w", err) - } - lines := strings.Split(string(contents), "\n") - inPath := false - for i, line := range lines { - if line != "" && !strings.HasPrefix(line, " ") { - inPath = line == "PATH" - } - if inPath && strings.HasPrefix(line, " remote: ") { - path := strings.TrimPrefix(line, " remote: ") - if !filepath.IsAbs(path) { - lines[i] = " remote: " + filepath.Join(root, path) - } - } - } - if err := os.WriteFile(filepath.Join(directory, "Gemfile.lock"), []byte(strings.Join(lines, "\n")), 0600); err != nil { - return fmt.Errorf("copy project lockfile: %w", err) - } - return nil -} - -func RubyEnvironment(gemfile string) map[string]string { - if gemfile == "" { - return nil - } - return map[string]string{ - "BUNDLE_GEMFILE": gemfile, - "BUNDLE_PATH": filepath.Join(filepath.Dir(gemfile), "gems"), - "BUNDLE_APP_CONFIG": filepath.Join(filepath.Dir(gemfile), "bundle-config"), - "BUNDLE_FROZEN": "false", "BUNDLE_DEPLOYMENT": "false", - "BUNDLE_WITH": "", "BUNDLE_WITHOUT": "", "BUNDLE_ONLY": "", - } -} From 53783162a6fc8cf014024a6d51680f893b10bd20 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 16:54:13 +0200 Subject: [PATCH 22/35] Avoid claiming an overridden Ruby tracer selection in reports --- internal/testdrive/testdrive.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index 14583496..b88d2856 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -190,6 +190,10 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { } tracerLabel := t.tracerLabel + if t.language == "ruby" { + // The project Gemfile may already pin the tracer selected by Bundler. + tracerLabel = "datadog-ci · isolated bundle" + } if installation.Project { tracerLabel = t.installedTracerLabel(t.projectTracer) + " · reused" } From 901440642a3895217daef32536fa62644b76c5c9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 17:01:51 +0200 Subject: [PATCH 23/35] Use the project Ruby bundle for testdrive --- internal/testdrive/command_test.go | 4 ++-- internal/testdrive/multiframework_test.go | 6 ++++-- internal/testdrive/ruby.go | 12 ++++++++++++ internal/testdrive/testdrive.go | 15 ++++++++------- 4 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 internal/testdrive/ruby.go diff --git a/internal/testdrive/command_test.go b/internal/testdrive/command_test.go index 7a0958db..f0052df3 100644 --- a/internal/testdrive/command_test.go +++ b/internal/testdrive/command_test.go @@ -47,10 +47,10 @@ func TestTestdrivePreservesExplicitCommandArguments(t *testing.T) { func TestTestdriveMinitestRequiresExecutableRailsBinstub(t *testing.T) { root := t.TempDir() + t.Chdir(root) require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0755)) require.NoError(t, os.WriteFile(filepath.Join(root, "bin", "rails"), []byte("#!/bin/sh\n"), 0644)) - command, args, err := TestdriveCommand(root, NewMinitest()) - require.NoError(t, err) + command, args := framework.NewMinitest().Command() require.Equal(t, "bundle", command) require.Equal(t, []string{"exec", "rake", "test"}, args) } diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 602ccb43..041c6091 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -44,6 +44,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { installer := &fakeTracer{preloadPath: filepath.Join(root, "isolated")} run.platform = installer run.projectTracer = "" + run.projectTracer = "" executor := &fakeTestdriveExecutor{} run.executor = executor run.startIntake = func(string) (localIntake, error) { @@ -66,7 +67,8 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { require.NotContains(t, executor.env, "NODE_OPTIONS") case "ruby": require.NotContains(t, executor.env, "BUNDLE_GEMFILE") - require.Contains(t, preview.String(), "Bundler updates the project Gemfile and lockfile") + require.Contains(t, preview.String(), "Bundler updates Gemfile and Gemfile.lock.") + require.NotContains(t, preview.String(), "It will not change") require.Contains(t, output.String(), "datadog-ci · installed in project") require.Contains(t, executor.env["RUBYOPT"], "datadog/ci/auto_instrument") require.NotContains(t, executor.env, "NODE_OPTIONS") @@ -128,7 +130,7 @@ func TestLanguageEnvironmentsPreserveCustomerOptions(t *testing.T) { require.Equal(t, "-q --ddtrace", python["PYTEST_ADDOPTS"]) ruby := (&Testdrive{language: "ruby"}).environment("/session/Gemfile", "http://127.0.0.1:1234", "session") require.True(t, strings.HasPrefix(ruby["RUBYOPT"], "-W0 ")) - require.NotContains(t, ruby, "BUNDLE_PATH") // Supplied by the platform installation result. + require.NotContains(t, ruby, "BUNDLE_PATH") // Inherit project Bundler configuration. } func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { diff --git a/internal/testdrive/ruby.go b/internal/testdrive/ruby.go new file mode 100644 index 00000000..b2deb334 --- /dev/null +++ b/internal/testdrive/ruby.go @@ -0,0 +1,12 @@ +package testdrive + +import ( + "os" + "strings" +) + +func rubyEnvironment(path string) map[string]string { + env := map[string]string{} + env["RUBYOPT"] = strings.TrimSpace(os.Getenv("RUBYOPT") + " -rbundler/setup -rdatadog/ci/auto_instrument") + return env +} diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index b88d2856..aab0773f 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -79,11 +79,6 @@ func Prepare(version string) (*Testdrive, error) { return nil, err } language := detectedPlatform.Name() - switch runner.Name() { - case "jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest": - default: - return nil, fmt.Errorf("testdrive does not yet support %s", runner.Name()) - } command, args := runner.Command() label := map[string]string{"javascript": "dd-trace", "python": "ddtrace", "ruby": "datadog-ci"}[language] + "@" + version @@ -140,6 +135,10 @@ func (t *Testdrive) Preview(output io.Writer) { } _, _ = fmt.Fprintf(output, " - run: %s\n", shellquote.Join(append([]string{command}, args...)...)) _, _ = fmt.Fprintln(output) + if t.language == "ruby" && t.projectTracer == "" { + _, _ = fmt.Fprintln(output, "Bundler updates Gemfile and Gemfile.lock.") + return + } patterns := map[string][]string{ "javascript": {"package.json", "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb"}, "python": {"pyproject.toml", "setup.py", "setup.cfg", "requirements*.txt", "requirements*.in", "Pipfile", "Pipfile.lock", "poetry.lock", "uv.lock", "pdm.lock", "pylock.toml"}, @@ -191,8 +190,8 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { tracerLabel := t.tracerLabel if t.language == "ruby" { - // The project Gemfile may already pin the tracer selected by Bundler. - tracerLabel = "datadog-ci · isolated bundle" + // Bundler owns the project dependency selection. + tracerLabel = "datadog-ci · installed in project" } if installation.Project { tracerLabel = t.installedTracerLabel(t.projectTracer) + " · reused" @@ -430,6 +429,8 @@ func (t *Testdrive) environment(path, intakeURL, sessionID string) map[string]st maps.Copy(env, t.javascriptEnvironment(path)) case "python": maps.Copy(env, pythonEnvironment(path)) + case "ruby": + maps.Copy(env, rubyEnvironment(path)) } return env } From 83f1a8846794468874df9c4fb705aa3cdb8b81a7 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:18:00 +0200 Subject: [PATCH 24/35] Guide multi-framework GitHub Actions onboarding --- internal/cmd/cmd.go | 15 +- internal/cmd/cmd_test.go | 35 ++-- internal/cmd/onboard.go | 29 +++ internal/cmd/onboard_test.go | 36 ++++ internal/onboard/instructions/github.md | 43 +++++ internal/onboard/multiframework_test.go | 56 ++++++ internal/onboard/onboard.go | 168 ++++++++++++++++++ internal/onboard/onboard_test.go | 105 +++++++++++ .../testdrive/frameworks_integration_test.go | 2 + 9 files changed, 469 insertions(+), 20 deletions(-) create mode 100644 internal/cmd/onboard.go create mode 100644 internal/cmd/onboard_test.go create mode 100644 internal/onboard/instructions/github.md create mode 100644 internal/onboard/multiframework_test.go create mode 100644 internal/onboard/onboard.go create mode 100644 internal/onboard/onboard_test.go diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 2af5b770..6f5a4c6a 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -31,9 +31,17 @@ import ( var defaultParallelism = settings.DefaultParallelism() var rootCmd = &cobra.Command{ - Use: "ddtest", - Short: "A test runner from Datadog", - Long: "Command line tool for running tests with Datadog Test Optimization.", + Use: "ddtest", + Short: "A test runner from Datadog", + Long: `Command line tool for running tests with Datadog Test Optimization. + +Start here: + + cd + ddtest onboard + +The onboard command detects the test and CI setup, explains the required +changes, points to local verification, and tells you when human action is needed.`, Version: buildinfo.CurrentVersion(), PersistentPreRunE: runPersistentPreRun, } @@ -154,6 +162,7 @@ func init() { rootCmd.AddCommand(planCmd) rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(onboardCmd) rootCmd.AddCommand(testdriveCmd) cobra.OnInitialize(settings.Init) diff --git a/internal/cmd/cmd_test.go b/internal/cmd/cmd_test.go index 3046ae8d..ce1383f7 100644 --- a/internal/cmd/cmd_test.go +++ b/internal/cmd/cmd_test.go @@ -274,23 +274,17 @@ func TestRootCommandFlags(t *testing.T) { } func TestCommandHierarchy(t *testing.T) { - // Verify that planCmd and runCmd are added to rootCmd + // Verify that the public commands are added to rootCmd. commands := rootCmd.Commands() - var foundPlan, foundRun bool + found := make(map[string]bool) for _, cmd := range commands { - if cmd.Name() == "plan" { - foundPlan = true - } - if cmd.Name() == "run" { - foundRun = true - } + found[cmd.Name()] = true } - if !foundPlan { - t.Error("plan command should be added to root command") - } - if !foundRun { - t.Error("run command should be added to root command") + for _, name := range []string{"onboard", "plan", "run", "testdrive"} { + if !found[name] { + t.Errorf("%s command should be added to root command", name) + } } } @@ -560,8 +554,15 @@ func TestExecute(t *testing.T) { } output := buf.String() - if !strings.Contains(output, "ddtest") { - t.Error("help output should contain command name 'ddtest'") + for _, expected := range []string{ + "Start here:", + "cd ", + "ddtest onboard", + "onboard Start here: onboard this repository to Test Optimization", + } { + if !strings.Contains(output, expected) { + t.Errorf("help output should contain %q:\n%s", expected, output) + } } } @@ -763,8 +764,8 @@ func TestCommandUsage(t *testing.T) { } // Expected commands (cobra adds completion and help automatically) - expectedCommands := []string{"plan", "run", "testdrive"} - requiredCommands := []string{"completion", "help [command]", "plan", "run", "testdrive"} + expectedCommands := []string{"onboard", "plan", "run", "testdrive"} + requiredCommands := []string{"completion", "help [command]", "onboard", "plan", "run", "testdrive"} // Verify minimum expected commands exist for _, expected := range expectedCommands { diff --git a/internal/cmd/onboard.go b/internal/cmd/onboard.go new file mode 100644 index 00000000..dd77d4a3 --- /dev/null +++ b/internal/cmd/onboard.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "io" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/spf13/cobra" +) + +var onboardCmd = newOnboardCommand(func(output io.Writer) error { + return onboard.Run(output) +}) + +func newOnboardCommand(run func(io.Writer) error) *cobra.Command { + return &cobra.Command{ + Use: "onboard", + Short: "Start here: onboard this repository to Test Optimization", + Long: "Detects the supported test setup and prints the smallest Datadog Test Optimization onboarding instructions. It does not edit files.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return run(cmd.OutOrStdout()) + }, + } +} diff --git a/internal/cmd/onboard_test.go b/internal/cmd/onboard_test.go new file mode 100644 index 00000000..5de56b54 --- /dev/null +++ b/internal/cmd/onboard_test.go @@ -0,0 +1,36 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "bytes" + "io" + "testing" +) + +func TestOnboardCommandRunsDetection(t *testing.T) { + called := false + command := newOnboardCommand(func(output io.Writer) error { + called = true + _, _ = io.WriteString(output, "onboarding instructions\n") + return nil + }) + if command.Short != "Start here: onboard this repository to Test Optimization" { + t.Fatalf("Short = %q", command.Short) + } + var output bytes.Buffer + command.SetOut(&output) + + if err := command.ExecuteContext(t.Context()); err != nil { + t.Fatalf("ExecuteContext() unexpected error: %v", err) + } + if !called { + t.Fatal("onboard command did not run detection") + } + if output.String() != "onboarding instructions\n" { + t.Fatalf("output = %q", output.String()) + } +} diff --git a/internal/onboard/instructions/github.md b/internal/onboard/instructions/github.md new file mode 100644 index 00000000..e75b8527 --- /dev/null +++ b/internal/onboard/instructions/github.md @@ -0,0 +1,43 @@ +# Enable Datadog Test Optimization for __FRAMEWORK__ on GitHub Actions + +Apply this setup to every GitHub Actions job that runs __FRAMEWORK__. + +## 1. Instrument the test job + +Add this step after checkout and dependency installation, immediately before the first test step: + +```yaml +- name: Configure Datadog Test Optimization + uses: datadog/test-visibility-github-action@v3 + with: + languages: __LANGUAGE__ + __TRACER_SETTING__ + api_key: ${{ secrets.DD_API_KEY }} + site: datadoghq.com +``` + +If the organization uses a Datadog site other than US1, replace `datadoghq.com` with that site. + +__BOOTSTRAP__ + +Keep the existing test command and unrelated workflow content unchanged. Add the Datadog action once per test job, not once per test step. + +## 2. Try it locally + +Run the local, credential-free setup check: + +```shell +ddtest testdrive +``` + +After it finishes, post every `Open report:` link printed by `ddtest testdrive` to the user so they can open the local Test Optimization report. Do not only summarize the result. + +## 3. Ask a human to connect Datadog + +The API key must be created and added to GitHub by a human. Ask the human to: + +1. Create a [Datadog API key](https://app.datadoghq.com/organization-settings/api-keys). +2. Add it to the GitHub repository as a secret named `DD_API_KEY`. +3. Tell you when the secret is ready without sharing the key itself. + +Do not ask the human to paste the API key into chat, and do not try to create or read the secret yourself. After the human confirms it is ready, commit and push the workflow change. The GitHub Actions run verifies the real Datadog backend connection. diff --git a/internal/onboard/multiframework_test.go b/internal/onboard/multiframework_test.go new file mode 100644 index 00000000..32913385 --- /dev/null +++ b/internal/onboard/multiframework_test.go @@ -0,0 +1,56 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOnboardAllSupportedFrameworks(t *testing.T) { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest", "rspec", "minitest"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + language := "js" + entry := name + files := map[string]string{"package.json": `{"scripts":{"test":"` + name + `"}}`} + switch name { + case "pytest": + files = map[string]string{"pyproject.toml": "[tool.pytest.ini_options]\n"} + language = "python" + entry = "uv run tox" + case "rspec", "minitest": + files = map[string]string{"Gemfile": "gem '" + name + "'\n"} + language = "ruby" + entry = "bundle exec rake" + } + for file, contents := range files { + require.NoError(t, os.WriteFile(filepath.Join(root, file), []byte(contents), 0644)) + } + require.NoError(t, os.MkdirAll(filepath.Join(root, ".github", "workflows"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".github", "workflows", "ci.yml"), []byte("jobs:\n tests:\n steps:\n - run: "+entry+"\n"), 0644)) + var output bytes.Buffer + t.Chdir(root) + require.NoError(t, Run(&output)) + require.Contains(t, output.String(), "languages: "+language) + require.Contains(t, output.String(), "ddtest testdrive --framework "+name) + require.Contains(t, output.String(), "Ask a human to connect Datadog") + if language != "js" { + require.NotContains(t, output.String(), "NODE_OPTIONS") + } + if name == "vitest" { + require.Contains(t, output.String(), "DD_TRACE_ESM_IMPORT") + } + if name == "cypress" { + require.Contains(t, output.String(), "setupNodeEvents") + } + }) + } +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go new file mode 100644 index 00000000..443584a2 --- /dev/null +++ b/internal/onboard/onboard.go @@ -0,0 +1,168 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +// Package onboard prints the smallest supported Test Optimization setup. +package onboard + +import ( + _ "embed" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/testdrive/tracer" +) + +const githubAction = "datadog/test-visibility-github-action" + +//go:embed instructions/github.md +var gitHubInstructions string + +// Run detects the first supported onboarding path and prints its instructions. +func Run(output io.Writer) error { + repositoryRoot, err := os.Getwd() + if err != nil { + return fmt.Errorf("find repository root: %w", err) + } + detectedPlatform, err := platform.DetectPlatform() + if err != nil { + return err + } + runner, err := detectedPlatform.DetectFramework() + if err != nil { + return err + } + language := detectedPlatform.Name() + name := runner.Name() + + workflows, configured, err := findWorkflows(repositoryRoot, language, name) + if err != nil { + return err + } + if len(workflows) == 0 { + return fmt.Errorf("onboard could not find a GitHub Actions workflow that runs %s", name) + } + + _, _ = fmt.Fprintf(output, "DDTest found %s, %s, and GitHub Actions.\n", language, name) + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, "Test workflow(s):") + for _, workflow := range workflows { + _, _ = fmt.Fprintf(output, " - %s\n", workflow) + } + + if len(configured) == len(workflows) { + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, "Datadog Test Optimization already appears in every detected test workflow.") + _, _ = fmt.Fprintf(output, "Run `ddtest testdrive --framework %s` to check the setup locally.\n", name) + _, _ = fmt.Fprintln(output, "After it finishes, post every `Open report:` link to the user so they can open the local Test Optimization report.") + return nil + } + + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, strings.TrimSpace(instructions(language, name))) + return nil +} + +func findWorkflows(repositoryRoot, language, name string) ([]string, []string, error) { + patterns := []string{ + filepath.Join(repositoryRoot, ".github", "workflows", "*.yml"), + filepath.Join(repositoryRoot, ".github", "workflows", "*.yaml"), + } + + var workflows []string + var configured []string + for _, pattern := range patterns { + matches, err := filepath.Glob(pattern) + if err != nil { + return nil, nil, fmt.Errorf("find GitHub Actions workflows: %w", err) + } + for _, path := range matches { + contents, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read %s: %w", path, err) + } + text := strings.ToLower(string(contents)) + if !looksLikeTestWorkflow(text, language, name) { + continue + } + + relativePath, err := filepath.Rel(repositoryRoot, path) + if err != nil { + return nil, nil, fmt.Errorf("make workflow path relative: %w", err) + } + relativePath = filepath.ToSlash(relativePath) + workflows = append(workflows, relativePath) + if strings.Contains(text, githubAction) { + configured = append(configured, relativePath) + } + } + } + + sort.Strings(workflows) + sort.Strings(configured) + return workflows, configured, nil +} + +func looksLikeTestWorkflow(workflow, language, name string) bool { + markers := []string{name, githubAction} + switch language { + case "javascript": + markers = append(markers, "npm test", "npm run test", "yarn test", "yarn run test", "pnpm test", "pnpm run test", "bun test", "bun run test") + case "ruby": + markers = append(markers, "bundle exec rake", "rake test", "rails test") + case "python": + markers = append(markers, "tox", "nox") + } + for _, marker := range markers { + if strings.Contains(workflow, marker) { + return true + } + } + return false +} + +func instructions(language, name string) string { + actionLanguage := language + var bootstrap, tracerSetting string + switch language { + case "javascript": + actionLanguage = "js" + tracerSetting = "js-tracer-version: " + tracer.JavaScriptVersion + bootstrap = javascriptBootstrap + if name == "cypress" { + bootstrap += "\n\n" + cypressBootstrap + } + if name == "cucumber" { + bootstrap += "\n\nFor the pinned tracer, also set DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED=false on the test step. This avoids a tracer crash on Cucumber Background/Rule nodes; basic reporting is unaffected." + } + case "python": + bootstrap = pythonBootstrap + tracerSetting = "python-tracer-version: " + tracer.PythonVersion + case "ruby": + bootstrap = rubyBootstrap + tracerSetting = "ruby-tracer-version: " + tracer.RubyVersion + } + text := strings.NewReplacer("__FRAMEWORK__", name, "__LANGUAGE__", actionLanguage, "__BOOTSTRAP__", bootstrap, "__TRACER_SETTING__", tracerSetting).Replace(gitHubInstructions) + return strings.ReplaceAll(text, "ddtest testdrive", "ddtest testdrive --framework "+name) +} + +const javascriptBootstrap = "Use Node.js 22 or newer. GitHub Actions cannot set NODE_OPTIONS for later steps, so merge this into the existing test step, preserving any current Node options:\n\n```yaml\nenv:\n NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}\n```\n\nThe --import loader is required for Vitest and other ESM tests." + +const pythonBootstrap = "The action exports PYTHONPATH and PYTEST_ADDOPTS=--ddtrace for pytest. Preserve these variables on the existing test step; do not replace its current arguments. Activate the same Python environment used for the tests before the action. If CI uses tox or nox, pass DD_*, PYTHONPATH, and PYTEST_ADDOPTS into the test environment." + +const rubyBootstrap = "The action installs datadog-ci into the CI bundle and exports RUBYOPT=-rbundler/setup -rdatadog/ci/auto_instrument. Preserve RUBYOPT on the RSpec or Minitest step and merge any existing Ruby options. Keep the existing bundle exec or binstub entry point." + +const cypressBootstrap = `Cypress also needs browser-side instrumentation; NODE_OPTIONS alone is insufficient: + +1. In the existing Cypress config, resolve the tracer root from the action's DD_TRACE_PACKAGE value: path.dirname(path.dirname(process.env.DD_TRACE_PACKAGE)). Only enable this configuration when that variable is present. +2. Compose the existing setupNodeEvents callback with require(path.join(tracerRoot, 'ci/cypress/plugin')). Preserve existing event handlers, including after:run and after:spec; do not replace them. +3. In setupNodeEvents, generate a support wrapper under RUNNER_TEMP. Write a literal require of path.join(tracerRoot, 'ci/cypress/support'), followed by a literal require of the existing resolved supportFile (unless it is false). Set the returned config.supportFile to that wrapper. Generating literal absolute imports lets Cypress's browser bundler resolve the isolated tracer without adding it to package.json. +4. Keep the project's existing application startup, browser installation, and cypress run command. Do not use cypress open. + +Use the same pattern for e2e or component configuration, whichever the selected job runs. The local testdrive creates equivalent wrappers inside its session automatically. See https://docs.datadoghq.com/tests/setup/javascript/ for the Cypress plugin contract.` diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go new file mode 100644 index 00000000..a0467878 --- /dev/null +++ b/internal/onboard/onboard_test.go @@ -0,0 +1,105 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunPrintsJestGitHubInstructions(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + test: + steps: + - run: npm test +`) + + var output bytes.Buffer + t.Chdir(repositoryRoot) + if err := Run(&output); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + for _, expected := range []string{ + "DDTest found javascript, jest, and GitHub Actions.", + ".github/workflows/test.yml", + "datadog/test-visibility-github-action@v3", + "api_key: ${{ secrets.DD_API_KEY }}", + "NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }}", + "ddtest testdrive", + "post every `Open report:` link", + "Ask a human to connect Datadog", + "without sharing the key itself", + } { + if !strings.Contains(output.String(), expected) { + t.Errorf("Run() output does not contain %q:\n%s", expected, output.String()) + } + } + if strings.Index(output.String(), "ddtest testdrive") > strings.Index(output.String(), "Ask a human to connect Datadog") { + t.Fatalf("API key setup must come after the local testdrive:\n%s", output.String()) + } +} + +func TestRunRecognizesExistingGitHubAction(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + test: + steps: + - uses: datadog/test-visibility-github-action@v3 + - run: npm test +`) + + var output bytes.Buffer + t.Chdir(repositoryRoot) + if err := Run(&output); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + if !strings.Contains(output.String(), "already appears in every detected test workflow") { + t.Fatalf("Run() did not recognize the existing setup:\n%s", output.String()) + } + if !strings.Contains(output.String(), "ddtest testdrive") { + t.Fatalf("Run() did not print the next step:\n%s", output.String()) + } + if !strings.Contains(output.String(), "post every `Open report:` link") { + t.Fatalf("Run() did not tell the agent to share the report:\n%s", output.String()) + } +} + +func TestRunRequiresGitHubJestWorkflow(t *testing.T) { + repositoryRoot := newJestRepository(t, "name: lint\njobs:\n lint:\n steps:\n - run: npm run lint\n") + + t.Chdir(repositoryRoot) + err := Run(&bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "could not find a GitHub Actions workflow that runs jest") { + t.Fatalf("Run() error = %v", err) + } +} + +func newJestRepository(t *testing.T, workflow string) string { + t.Helper() + repositoryRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(repositoryRoot, "package.json"), []byte(`{ + "scripts": {"test": "jest"}, + "devDependencies": {"jest": "30.0.0"} +}`), 0644); err != nil { + t.Fatal(err) + } + workflowDirectory := filepath.Join(repositoryRoot, ".github", "workflows") + if err := os.MkdirAll(workflowDirectory, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workflowDirectory, "test.yml"), []byte(workflow), 0644); err != nil { + t.Fatal(err) + } + return repositoryRoot +} diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index 05b0f6f8..a78d4e06 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -78,6 +78,8 @@ func TestPublicFrameworkTestdrives(t *testing.T) { before[name] = string(contents) } } + onboard := integrationCommand(t, ctx, root, env, binary, "onboard") + require.Contains(t, onboard, "datadog/test-visibility-github-action@v3") output := integrationCommand(t, ctx, root, env, binary, "testdrive", "--yes") require.Contains(t, output, "Test events received.") require.Contains(t, output, "Open report:") From 3111dc611ad6bb69bce3d5bc66b0a4832745db92 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 09:17:05 +0200 Subject: [PATCH 25/35] Validate onboarding setup per test job --- go.mod | 2 +- internal/onboard/instructions/github.md | 2 +- internal/onboard/onboard.go | 113 ++++++++++++++++++------ internal/onboard/onboard_test.go | 41 +++++++++ 4 files changed, 127 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index da7a067d..606ac794 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/tinylib/msgp v1.6.5 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 + go.yaml.in/yaml/v3 v3.0.5 ) require ( @@ -27,6 +28,5 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/text v0.42.0 // indirect ) diff --git a/internal/onboard/instructions/github.md b/internal/onboard/instructions/github.md index e75b8527..be45402b 100644 --- a/internal/onboard/instructions/github.md +++ b/internal/onboard/instructions/github.md @@ -36,7 +36,7 @@ After it finishes, post every `Open report:` link printed by `ddtest testdrive` The API key must be created and added to GitHub by a human. Ask the human to: -1. Create a [Datadog API key](https://app.datadoghq.com/organization-settings/api-keys). +1. Follow the site-neutral [Datadog API key instructions](https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token) and create the key in the selected Datadog site. 2. Add it to the GitHub repository as a secret named `DD_API_KEY`. 3. Tell you when the secret is ready without sharing the key itself. diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index 443584a2..82c99cee 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/testdrive/tracer" + "go.yaml.in/yaml/v3" ) const githubAction = "datadog/test-visibility-github-action" @@ -70,37 +71,42 @@ func Run(output io.Writer) error { } func findWorkflows(repositoryRoot, language, name string) ([]string, []string, error) { - patterns := []string{ - filepath.Join(repositoryRoot, ".github", "workflows", "*.yml"), - filepath.Join(repositoryRoot, ".github", "workflows", "*.yaml"), - } - var workflows []string var configured []string - for _, pattern := range patterns { - matches, err := filepath.Glob(pattern) + directory := filepath.Join(repositoryRoot, ".github", "workflows") + entries, err := os.ReadDir(directory) + if os.IsNotExist(err) { + return workflows, configured, nil + } + if err != nil { + return nil, nil, fmt.Errorf("find GitHub Actions workflows: %w", err) + } + for _, entry := range entries { + if entry.IsDir() || (filepath.Ext(entry.Name()) != ".yml" && filepath.Ext(entry.Name()) != ".yaml") { + continue + } + path := filepath.Join(directory, entry.Name()) + contents, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read %s: %w", path, err) + } + text := strings.ToLower(string(contents)) + if !looksLikeTestWorkflow(text, language, name) { + continue + } + + relativePath, err := filepath.Rel(repositoryRoot, path) + if err != nil { + return nil, nil, fmt.Errorf("make workflow path relative: %w", err) + } + relativePath = filepath.ToSlash(relativePath) + workflows = append(workflows, relativePath) + isConfigured, err := allTestJobsConfigured(contents, language, name) if err != nil { - return nil, nil, fmt.Errorf("find GitHub Actions workflows: %w", err) + return nil, nil, fmt.Errorf("parse %s: %w", path, err) } - for _, path := range matches { - contents, err := os.ReadFile(path) - if err != nil { - return nil, nil, fmt.Errorf("read %s: %w", path, err) - } - text := strings.ToLower(string(contents)) - if !looksLikeTestWorkflow(text, language, name) { - continue - } - - relativePath, err := filepath.Rel(repositoryRoot, path) - if err != nil { - return nil, nil, fmt.Errorf("make workflow path relative: %w", err) - } - relativePath = filepath.ToSlash(relativePath) - workflows = append(workflows, relativePath) - if strings.Contains(text, githubAction) { - configured = append(configured, relativePath) - } + if isConfigured { + configured = append(configured, relativePath) } } @@ -109,6 +115,55 @@ func findWorkflows(repositoryRoot, language, name string) ([]string, []string, e return workflows, configured, nil } +func allTestJobsConfigured(contents []byte, language, name string) (bool, error) { + var workflow struct { + Jobs map[string]struct { + Steps []struct { + Run string `yaml:"run"` + Uses string `yaml:"uses"` + } `yaml:"steps"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(contents, &workflow); err != nil { + return false, err + } + foundTestJob := false + for _, job := range workflow.Jobs { + var commands []string + configured := false + for _, step := range job.Steps { + commands = append(commands, strings.ToLower(step.Run)) + configured = configured || strings.Contains(strings.ToLower(step.Uses), githubAction) + } + if !looksLikeTestJob(strings.Join(commands, "\n"), language, name) { + continue + } + foundTestJob = true + if !configured { + return false, nil + } + } + return foundTestJob, nil +} + +func looksLikeTestJob(commands, language, name string) bool { + markers := []string{name} + switch language { + case "javascript": + markers = append(markers, "npm test", "npm run test", "yarn test", "yarn run test", "pnpm test", "pnpm run test", "bun test", "bun run test") + case "ruby": + markers = append(markers, "bundle exec rake", "rake test", "rails test") + case "python": + markers = append(markers, "tox", "nox") + } + for _, marker := range markers { + if strings.Contains(commands, marker) { + return true + } + } + return false +} + func looksLikeTestWorkflow(workflow, language, name string) bool { markers := []string{name, githubAction} switch language { @@ -160,8 +215,8 @@ const rubyBootstrap = "The action installs datadog-ci into the CI bundle and exp const cypressBootstrap = `Cypress also needs browser-side instrumentation; NODE_OPTIONS alone is insufficient: -1. In the existing Cypress config, resolve the tracer root from the action's DD_TRACE_PACKAGE value: path.dirname(path.dirname(process.env.DD_TRACE_PACKAGE)). Only enable this configuration when that variable is present. -2. Compose the existing setupNodeEvents callback with require(path.join(tracerRoot, 'ci/cypress/plugin')). Preserve existing event handlers, including after:run and after:spec; do not replace them. +1. In the existing Cypress config, resolve the tracer root from the action's DD_TRACE_PACKAGE value: path.dirname(path.dirname(process.env.DD_TRACE_PACKAGE)). Only enable this configuration when that variable is present. Use import syntax in an ESM config and require in a CommonJS config. +2. Compose the existing setupNodeEvents callback with the module at path.join(tracerRoot, 'ci/cypress/plugin'), loaded with import in ESM or require in CommonJS. Preserve existing event handlers, including after:run and after:spec; do not replace them. 3. In setupNodeEvents, generate a support wrapper under RUNNER_TEMP. Write a literal require of path.join(tracerRoot, 'ci/cypress/support'), followed by a literal require of the existing resolved supportFile (unless it is false). Set the returned config.supportFile to that wrapper. Generating literal absolute imports lets Cypress's browser bundler resolve the isolated tracer without adding it to package.json. 4. Keep the project's existing application startup, browser installation, and cypress run command. Do not use cypress open. diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go index a0467878..e7c41498 100644 --- a/internal/onboard/onboard_test.go +++ b/internal/onboard/onboard_test.go @@ -75,6 +75,47 @@ jobs: } } +func TestRunRequiresActionInEveryTestJob(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + unit: + steps: + - uses: datadog/test-visibility-github-action@v3 + - run: npm test + integration: + steps: + - run: npm test +`) + var output bytes.Buffer + t.Chdir(repositoryRoot) + if err := Run(&output); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), "already appears in every detected test workflow") { + t.Fatalf("Run() treated a partially configured workflow as complete:\n%s", output.String()) + } +} + +func TestRunTreatsRepositoryRootAsLiteralPath(t *testing.T) { + parent := t.TempDir() + repositoryRoot := filepath.Join(parent, "project[old]") + if err := os.Mkdir(repositoryRoot, 0755); err != nil { + t.Fatal(err) + } + fixture := newJestRepository(t, "name: tests\njobs:\n test:\n steps:\n - run: npm test\n") + if err := os.Rename(filepath.Join(fixture, "package.json"), filepath.Join(repositoryRoot, "package.json")); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(fixture, ".github"), filepath.Join(repositoryRoot, ".github")); err != nil { + t.Fatal(err) + } + t.Chdir(repositoryRoot) + if err := Run(&bytes.Buffer{}); err != nil { + t.Fatalf("Run() failed for literal repository path: %v", err) + } +} + func TestRunRequiresGitHubJestWorkflow(t *testing.T) { repositoryRoot := newJestRepository(t, "name: lint\njobs:\n lint:\n steps:\n - run: npm run lint\n") From 878b3529140704a5f0331fff09bff0c1dd1cf62a Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 13:39:34 +0200 Subject: [PATCH 26/35] Recommend latest tracers in onboarding setup --- internal/onboard/multiframework_test.go | 1 + internal/onboard/onboard.go | 10 ++++------ internal/testdrive/multiframework_test.go | 1 + 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/onboard/multiframework_test.go b/internal/onboard/multiframework_test.go index 32913385..fa96d902 100644 --- a/internal/onboard/multiframework_test.go +++ b/internal/onboard/multiframework_test.go @@ -40,6 +40,7 @@ func TestOnboardAllSupportedFrameworks(t *testing.T) { t.Chdir(root) require.NoError(t, Run(&output)) require.Contains(t, output.String(), "languages: "+language) + require.Contains(t, output.String(), language+"-tracer-version: '' # Latest release") require.Contains(t, output.String(), "ddtest testdrive --framework "+name) require.Contains(t, output.String(), "Ask a human to connect Datadog") if language != "js" { diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index 82c99cee..1bd97176 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -16,7 +16,6 @@ import ( "strings" "github.com/DataDog/ddtest/internal/platform" - "github.com/DataDog/ddtest/internal/testdrive/tracer" "go.yaml.in/yaml/v3" ) @@ -184,25 +183,24 @@ func looksLikeTestWorkflow(workflow, language, name string) bool { func instructions(language, name string) string { actionLanguage := language - var bootstrap, tracerSetting string + var bootstrap string switch language { case "javascript": actionLanguage = "js" - tracerSetting = "js-tracer-version: " + tracer.JavaScriptVersion bootstrap = javascriptBootstrap if name == "cypress" { bootstrap += "\n\n" + cypressBootstrap } if name == "cucumber" { - bootstrap += "\n\nFor the pinned tracer, also set DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED=false on the test step. This avoids a tracer crash on Cucumber Background/Rule nodes; basic reporting is unaffected." + bootstrap += "\n\nFor Cucumber, also set DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED=false on the test step. This avoids a tracer crash on Cucumber Background/Rule nodes; basic reporting is unaffected." } case "python": bootstrap = pythonBootstrap - tracerSetting = "python-tracer-version: " + tracer.PythonVersion case "ruby": bootstrap = rubyBootstrap - tracerSetting = "ruby-tracer-version: " + tracer.RubyVersion } + // Empty inputs override the action's pinned defaults and request the latest release. + tracerSetting := actionLanguage + "-tracer-version: '' # Latest release" text := strings.NewReplacer("__FRAMEWORK__", name, "__LANGUAGE__", actionLanguage, "__BOOTSTRAP__", bootstrap, "__TRACER_SETTING__", tracerSetting).Replace(gitHubInstructions) return strings.ReplaceAll(text, "ddtest testdrive", "ddtest testdrive --framework "+name) } diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 041c6091..9c291e18 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -45,6 +45,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { run.platform = installer run.projectTracer = "" run.projectTracer = "" + run.projectTracer = "" executor := &fakeTestdriveExecutor{} run.executor = executor run.startIntake = func(string) (localIntake, error) { From bc77e8086fcc9b5ad73ed365bf679c3a1f239493 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 22 Sep 2026 13:18:07 +0200 Subject: [PATCH 27/35] Document onboarding release and OSS validation --- Makefile | 9 +- README.md | 56 +++++- docs/design/agent-driven-onboarding.md | 231 +++++++++++++++++++++++++ docs/testing/onboarding-milestone-2.md | 183 ++++++++++++++++++++ 4 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 docs/design/agent-driven-onboarding.md create mode 100644 docs/testing/onboarding-milestone-2.md diff --git a/Makefile b/Makefile index a63e19a2..9deb7588 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ .DEFAULT_GOAL := build -.PHONY: clean fmt vet lint build run release +.PHONY: clean fmt vet lint test build install run release VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X github.com/DataDog/ddtest/internal/buildinfo.Version=$(VERSION) +GOBIN ?= $(shell go env GOBIN) +ifeq ($(strip $(GOBIN)),) +GOBIN := $(shell go env GOPATH)/bin +endif clean: go clean -i -x fmt: @@ -14,6 +18,9 @@ test: go test ./... build: test lint go build -ldflags="$(LDFLAGS)" -o ddtest main.go +install: + GOBIN="$(GOBIN)" go install -ldflags="$(LDFLAGS)" . + @echo "Installed ddtest to $(GOBIN)/ddtest" run: go run main.go release: diff --git a/README.md b/README.md index 4b193f7a..3d887f6e 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,63 @@ Currently supported: - Python with pytest. - JavaScript with Cucumber, Cypress, Jest, Mocha, Playwright, or Vitest. +## Try Test Optimization locally + +From a supported repository with its test dependencies installed: + +```sh +ddtest onboard +ddtest testdrive +``` + +`onboard` prints GitHub Actions setup instructions. `testdrive` previews its +commands, asks for confirmation, and runs your own tests against a local intake. +It needs no Datadog account, API key, or Agent. Non-interactive callers can review +the preview, then use `ddtest testdrive --yes`. + +All nine frameworks listed above are supported. If a repository contains several +runners, select one with `--framework`. Use `--command` to select a custom entry +point or a small representative part of a large suite: + +```sh +ddtest onboard --framework playwright +ddtest testdrive --framework playwright --command 'npm run test:e2e -- --project=chromium' --yes +``` + +The terminal links to a self-contained HTML report, decoded JSON traffic, and +complete test output under `.testoptimization/testdrive//`. Reports +separate instrumentation success from failed tests, and explicitly indicate when +coverage was not reported. Tracer configuration errors are shown separately. +The local intake supports agentless traffic; Agent/EVP routing is not supported. +Receiving events does not verify test skipping, EFD, or Test Management behavior. +Keep this directory out of source control. Each run +has its own installation, files, and loopback port. + +Local testdrive prerequisites: + +- JavaScript: Node.js 22+, npm for the isolated `dd-trace@6.15.0` installation, + and the project's package manager and dependencies. Vitest/ESM loading and + Cypress config/support wrappers are supplied automatically. +- Python: an activated project environment with pytest and pip. `ddtrace==4.15.1` + is installed into a session-owned directory; the active environment is unchanged. +- Ruby: Ruby/Bundler and native gem build tools. A session-owned bundle adds + `datadog-ci@1.39.0`, preserving the original Gemfile and lockfile. The pinned + tracer's native extensions require a checkout path without spaces. +- Browser suites: install the project's browsers and start any required services + first, or use its existing test command that manages them. Testdrive does not + install browsers or start applications on its own. + +Project dependency manifests and lockfiles are not edited by testdrive. Testdrive +uses the framework’s normal command; pass `--command` to run a package script and +its lifecycle hooks or other custom setup. Tracer downloads require network access. This release covers root projects and GitHub Actions onboarding; +monorepo orchestration and other CI providers are outside this scope. + +See the [Milestone 2 validation record](docs/testing/onboarding-milestone-2.md) +for tested repository commits, commands, and limitations. + ## Prerequisites -Before using DDTest, you must have **Datadog Test Optimization** already set up and enabled with a Datadog Test Optimization library for your language and framework. DDTest relies on this integration to discover your tests and plan test execution accordingly. +Before using `ddtest plan` or `ddtest run`, you must have **Datadog Test Optimization** already set up and enabled with a Datadog Test Optimization library for your language and framework. DDTest relies on this integration to discover your tests and plan test execution accordingly. Minimum supported library and runtime requirements: diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md new file mode 100644 index 00000000..86cbfe1d --- /dev/null +++ b/docs/design/agent-driven-onboarding.md @@ -0,0 +1,231 @@ +# Agent-driven onboarding + +Status: Milestones 0, 1, and 2 implemented; Milestones 3 and 4 proposed + +Last updated: 2026-09-22 + +## Goal + +A user should be able to give a coding agent one prompt: + +> Onboard test optimization using ddtest. + +The agent discovers the flow through `ddtest help`, makes the small CI edit suggested by `ddtest onboard`, runs `ddtest testdrive`, and gives the user a clickable report showing their own tests. + +The local testdrive requires no Datadog account, API key, or Agent. Credentials come last, are configured by a human in CI, and are never shown to the coding agent. + +## Product rules + +- Use the customer's real test suite, not a demo. +- Keep the main flow to `ddtest onboard` and `ddtest testdrive`. +- Detect the repository instead of asking setup questions. +- Show commands and filesystem changes before running them. +- Keep tracer installation isolated from project manifests and lockfiles. +- Explain what worked, what did not, and the next useful action. +- Give the user clickable local and CI links. +- Dogfood a thin end-to-end slice before adding abstractions. + +For now, do not build a generic onboarding framework, YAML editor, public result schema, compatibility matrix, or exhaustive error taxonomy. + +## What exists today + +The working preview is in draft PR #128 on `anmarchenko/agentic-onboarding-runbook`. + +### User flow + +- `ddtest help` points to `ddtest onboard`. +- `ddtest onboard` detects all nine supported platform/framework pairs and candidate GitHub Actions test workflows; the coding agent inspects the actual jobs. Repositories with several frameworks use `--framework`. +- It prints repository-owned Markdown instructions containing the concrete workflow edit, asks the agent to run `ddtest testdrive`, and tells the agent to post every report link to the user. +- The local testdrive comes before the human API-key and GitHub-secret steps. +- `ddtest testdrive` previews its commands and file changes. Interactive terminals ask once for confirmation. Non-interactive callers must review the preview and rerun with `--yes`. +- `make install` builds and installs DDTest into the current user's Go bin directory on macOS and Linux. + +### Local testdrive + +- Each run has a unique directory and kernel-assigned loopback port, so sessions can run concurrently. +- Pinned tracers (`dd-trace@6.15.0`, `ddtrace==4.15.1`, and `datadog-ci@1.39.0`) are installed inside each session. Python keeps the active interpreter; Ruby uses an overlay bundle and a copy of the project lockfile. Project dependency files are untouched. +- The local intake supports the endpoints exercised by that tracer and enables Test Optimization, coverage, Intelligent Test Runner, Early Flake Detection, Auto Test Retries, Impacted Tests, failed-test replay, and Test Management. +- Test events and test- or suite-level coverage are decoded. Raw multipart or msgpack payloads are not retained; saved traffic is JSON only. +- Complete test output is saved separately. + +The terminal and self-contained HTML report answer: + +- Did any tests fail? +- Are any tests flaky because they passed on retry? +- Are any tests unusually slow compared with the median? +- Do any tests or suites cover unusually many files compared with the median? + +Problem cards appear only when the answer is yes. Affected tests are always visible and expand to show attempts, timings, errors, retry information, source excerpts, and coverage. Flaky tests use `test.final_status` and are not also reported as failed. Covered files appear one per line and are paginated 50 at a time. Separate paginated tabs list all suites and tests. + +The report also shows event and coverage counts, framework result, tracer version, isolated installation, saved JSON traffic, and test output. `testdrive` prints it as an absolute clickable `file://` link. + +### Code map + +- `internal/onboard/`: repository detection and embedded Markdown instructions. +- `internal/testdrive/`: session lifecycle, preview, execution, terminal output, and HTML report. +- `internal/testdrive/tracer/`: isolated tracer installation behind the `Tracer` interface. +- `internal/testdrive/intake/`: local intake, JSON capture, event and coverage decoding, and findings. +- `internal/platform/` and `internal/framework/`: existing detection and test commands that future milestones should reuse. + +### Evidence and limits + +- Integration tests run the public CLI with real pinned tracers against fixtures for all nine frameworks. Coverage is reported only when supplied by the tracer. +- A concurrent integration test proves port, traffic, and file isolation. +- Unit tests cover decoding, final status, flaky tests, both coverage granularities, medians, source excerpts, HTML rendering, and confirmation behavior. +- Dogfooding on React Native Paper recognized 1,363 events and coverage for all 680 logical tests, including Early Flake Detection retries. + +Public support covers all nine pairs in Milestone 2, with GitHub Actions onboarding. The intake is not a complete Datadog backend. There is no upload service, Datadog forwarding, stable JSON contract, or savings calculation. Onboarding instructions are copied from the onboarding MCP source rather than shared with it. + +Every change must pass: + +```shell +make test +make lint +``` + +## Milestone 0: prove the local loop — complete + +Milestone 0 proved the risky path before designing the product around it: + +- start a minimal local Test Optimization intake on port `0`; +- install and run a real pinned JavaScript tracer; +- receive a real Jest test event and coverage; +- save each run in its own session directory; +- run two sessions concurrently without collisions; +- require no Datadog credentials or Agent. + +## Milestone 1: delightful Jest preview — implemented + +Milestone 1 turned the spike into the current `onboard` and `testdrive` flow described above. + +Its important interaction contract is: + +- detection and preview happen before any write or external command; +- `ddtest testdrive --yes` is the explicit non-interactive path; +- running the command is one decision—there is no persisted plan, checksum, approval file, or second execution command; +- instrumentation success is independent of whether customer tests pass; +- the project manifest and lockfile are never changed; +- the agent posts the report link to the user. + +Keep dogfooding Jest while later milestones are built. Fix repeated real problems directly; extract shared types only when another working implementation needs them. + +## Milestone 2: every supported platform/framework pair — implemented + +Extend the same basic onboarding and testdrive to the pairs already supported by DDTest: + +| Platform | Frameworks | +| --- | --- | +| JavaScript | Jest, Mocha, Cypress, Playwright, Cucumber, Vitest | +| Python | pytest | +| Ruby | RSpec, Minitest | + +Implemented in vertical slices: + +1. Reuse each platform and framework's `Detect` method and test command. Remove Jest-specific names from the shared report. +2. Add the remaining JavaScript frameworks using the existing isolated `dd-trace` installation. +3. Add one pinned isolated `ddtrace` installation for pytest. +4. Add one pinned isolated Ruby tracer installation shared by RSpec and Minitest. +5. Add a tiny real-tracer fixture for every pair and dogfood at least one real repository per platform. + +For every pair, `onboard` finds the relevant GitHub Actions job and prints one small setup. `testdrive` previews and runs the detected framework's normal command (or the explicit `--command` entry point), treats received events as proof even when tests fail, reports missing coverage honestly, and preserves the same terminal and HTML experience where the tracer supplies the data. + +Do not solve monorepos, new CI providers, or cross-platform tracer abstractions here. One known-good tracer version per platform is enough. + +All nine pairs have completed the credential-free flow with JSON traffic and local reports. Manifests and lockfiles are checked for preservation. The validation record distinguishes full-suite runs from representative browser/RSpec subsets and records the pinned Cucumber tracer workaround. + +## Milestone 3: guided Test Parallelization onboarding + +Starting prompt: + +> Onboard test parallelization using ddtest. + +Add `ddtest onboard parallelization` as the obvious next step after Test Optimization works. If Test Optimization is not configured, point back to `ddtest onboard`; do not combine both migrations. + +The command detects the platform, framework, and GitHub Actions test job, then tells the coding agent how to make one concrete transformation using the existing product: + +1. A plan job installs the project as CI already does and runs `ddtest plan`. +2. The plan job exposes DDTest's generated matrix and uploads `.testoptimization/`. +3. A matrix job downloads the artifact and runs `ddtest run --ci-node ${{ matrix.ci_node_index }}`. +4. The old command is removed so CI does not run the full suite twice. + +Start with one worker per CI node, `fail-fast: false`, explicit minimum and maximum parallelism, and the existing CI-job overhead model. Preserve runtime setup, environment, services, caches, permissions, timeouts, and artifacts. DDTest prints instructions; the agent edits the workflow. Do not build a YAML rewriting engine or another planner. + +The first real GitHub Actions run is the test drive. The agent gives the user its link and reports: + +- whether the plan job and every node passed; +- selected node count; +- estimated full-suite and parallel wall time; +- modeled CI overhead and imbalance; +- dedicated slow-suite runners, if any; +- the smallest corrective edit for a concrete setup failure. + +Selecting one node is a valid success when extra nodes would not help enough. Support the same nine pairs as Milestone 2. Stop clearly when an existing matrix or parallel runner cannot be combined safely. + +Milestone 3 ships when an agent can discover the flow from the starting prompt, make a reviewable GitHub Actions edit, run the existing planner and runner, and return a clickable CI link with the important plan facts. + +## Milestone 4: `dd-trace-js` runbook parity + +Match the useful conclusions of the validation runbook, not its internal architecture. Keep `ddtest onboard` and `ddtest testdrive`; do not copy its manifest, execution-plan, checksum, approval-file, persisted-lock, or exit-code machinery. + +The terminal and local report show five independent conclusions: + +- **Basic Reporting:** the tracer reports a real project test. +- **CI configuration:** the selected job visibly initializes Test Optimization and configures transport. +- **Early Flake Detection:** a new passing test is retried with the expected reason. +- **Auto Test Retries:** a fail-once test passes on retry with the expected reason. +- **Test Management:** a configured test is matched and tagged as quarantined. + +Each conclusion is simply works, needs attention, or could not be checked. Name the exact missing prerequisite, first useful action, and cleanup status. Keep this validation scope separate from code-coverage counts. + +Implementation order: + +1. Add all five conclusions to the simplest Jest repository and dogfood the complete flow before generalizing it. +2. Keep the normal happy path: the customer's instrumented suite proves Basic Reporting. +3. After Basic Reporting succeeds, run small DDTest-owned tests for the three advanced features. The local intake supplies settings, known tests, and managed tests; emitted attempts and events must prove behavior. +4. If Basic Reporting is inconclusive, compare one representative test without and with instrumentation and debug logging. Do not double every successful testdrive. +5. Audit the selected GitHub Actions job without executing it. Resolve the setup DDTest generates, direct commands, and simple local package scripts; report dynamic or remote wrappers as inconclusive. +6. Extend the proven Jest slice to Mocha, Cypress, Playwright, Cucumber, and Vitest. + +Temporary tests appear in the preview, are created only after confirmation, and are removed after the run. Their decoded JSON events and output remain in the session. Browser- or application-backed checks may be inconclusive with the missing prerequisite named; DDTest does not start applications or install browsers implicitly. + +Python and Ruby retain Milestone 2's Basic Reporting and suite analysis until their tracer behavior and real dogfood cases justify equivalent advanced checks. + +Milestone 4 ships when all six JavaScript frameworks report the five conclusions without Datadog credentials, preserve project dependencies and concurrent-session isolation, clean up temporary tests, and provide clickable local and CI links. + +## Next work + +Milestone 2 now has public-CLI real-tracer fixtures for every pair and independent open-source dogfood runs. See [the validation record](../testing/onboarding-milestone-2.md). The next planned feature is Milestone 3; keep the existing nine-pair flow green. + +For each slice: + +1. add one real-tracer fixture; +2. run it in a real repository; +3. record what the human or agent had to guess; +4. fix observed friction; +5. run `make test`, `make lint`, and the installed binary. + +## Later ideas + +These should not delay Milestones 2–4: + +- Homebrew distribution. +- One offline source for the Markdown instructions currently duplicated with the onboarding MCP implementation in `dd-source`. +- A broad `ddeval` suite built from real repository and CI shapes. +- Fully local TIA backed by SQLite coverage history, considering committed, staged, unstaged, and untracked changes. +- Real Datadog mode when `DD_API_KEY` is present. +- Local reproduction of CI operating-system and runtime tags. +- Historical test analysis through the Datadog API. +- `ddtest doctor` as a reusable diagnostic command if the integrated flow proves it is needed. +- More CI providers and monorepo orchestration. +- Multiple tracer-version support. +- Local savings estimates and historical replay. +- A hosted Testdog page for sharing a report without a Datadog account. + +Choose the next slice from what users struggle with after Milestone 4, not from a speculative architecture. + +## References + +- `~/p/shepherd/tools/mockdog`: local Test Optimization intake precedent. +- `~/p/dd-trace-js/ci/runbook.md`: validation conclusions and JavaScript adapter behavior. +- `~/p/test-visibility-install-script`: isolated JavaScript tracer installation precedent. +- The onboarding MCP instructions in `~/dd/dd-source`: current source material for CI setup instructions. diff --git a/docs/testing/onboarding-milestone-2.md b/docs/testing/onboarding-milestone-2.md new file mode 100644 index 00000000..074c3610 --- /dev/null +++ b/docs/testing/onboarding-milestone-2.md @@ -0,0 +1,183 @@ +# Milestone 2 validation — 2026-09-22 + +All nine supported frameworks completed the public `onboard → testdrive --yes → +report.html` flow on independent open-source repositories. Both commands exited +zero for every row below. Each run received real tracer events, retained decoded +JSON traffic and test output, and generated a local HTML report. + +This was a manually selected open-source sample, not a statistically random +compatibility study. Browser and RSpec checks used representative subsets as +identified below. These results do not imply every version or configuration of +each framework is compatible. + +## Environment and checks + +- macOS arm64; Go 1.26.5; Node.js 24.14.0; Python 3.11; Ruby 3.4.7. +- Pinned tracers: `dd-trace@6.15.0`, `ddtrace==4.15.1`, `datadog-ci@1.39.0`. +- `make test` and `make lint` pass. +- The opt-in public CLI matrix runs real fixtures for all nine frameworks, + including paths containing spaces for JavaScript/Python, existing Ruby + lockfiles, a Cucumber Background, and preserved Cypress task/after-run hooks. +- Existing instrumented Jest and concurrent-session isolation checks pass. +- Root manifests and lockfiles were hashed immediately before and after each + open-source run, including presence/absence. All nine snapshots were unchanged. + Dependency installation and application builds happened before those snapshots. +- No Datadog credentials or Agent were used. This validates local instrumentation + and generated onboarding guidance, not a connection to a live Datadog account. + +## Open-source results + +“Covered” means a test had associated coverage in received traffic. Zero means +coverage was not reported, not that the test covered no application code. Events +can exceed logical tests because the local intake enables retries. + +| Framework | Repository / exact commit | Scope | Logical tests | Events | Covered | +| --- | --- | --- | ---: | ---: | ---: | +| jest | [typestack/class-validator](https://github.com/typestack/class-validator/tree/2e1a5c27dbd65b80e27fe96b49bd6e6641fa3603) | Full Jest suite | 743 | 1549 | 743 | +| mocha | [omichelsen/compare-versions](https://github.com/omichelsen/compare-versions/tree/98e81116ef4197b42dca8c3fde8d1e8166e2a81f) | Full Mocha suite | 336 | 678 | 0 | +| vitest | [unjs/destr](https://github.com/unjs/destr/tree/541b6f9aeada9fc30de9c5a7e086dbfc1c6fcdc7) | Full Vitest suite (lint omitted) | 22 | 44 | 22 | +| playwright | [dropbox/ttvc](https://github.com/dropbox/ttvc/tree/239481cdecb841f63f0b3f3e13be52275333b18d) | One Chromium browser test | 1 | 2 | 0 | +| cypress | [cypress-io/cypress-example-kitchensink](https://github.com/cypress-io/cypress-example-kitchensink/tree/ddaaa92080b68d71d7a1797b4ed20ada18ed2a2a) | Six Todo browser tests | 6 | 12 | 0 | +| cucumber | [cloudevents/sdk-javascript](https://github.com/cloudevents/sdk-javascript/tree/bf5d53f2862248d72d9869cf5173f51b32583faf) | HTTP/Kafka conformance suite | 7 | 14 | 0 | +| pytest | [pallets/itsdangerous](https://github.com/pallets/itsdangerous/tree/672971d66a2ef9f85151e53283113f33d642dabd) | Full pytest suite | 297 | 297 | 297 | +| rspec | [ruby-concurrency/concurrent-ruby](https://github.com/ruby-concurrency/concurrent-ruby/tree/e674fb2688206bb7cb66dee108e1d08184413afb) | AtomicBoolean spec file | 25 | 25 | 25 | +| minitest | [ruby-i18n/i18n](https://github.com/ruby-i18n/i18n/tree/547917dd8d41fab781a81880f22687fc4eac5d85) | Full Minitest suite | 1607 | 1607 | 1607 | + +## Reproduction + +Build DDTest with Go 1.26.5 and put the resulting binary on PATH. Check out each +linked commit and install that project's dependencies first. Use Node.js 22+ for +the JavaScript tracer and activate the project's Python environment for pytest. +The Ruby checks use Ruby 3.4.7 with native extension build tools. + +Repository preparation used: + +- class-validator and compare-versions: `npm ci --no-audit --no-fund`. +- destr: `pnpm install --frozen-lockfile` (pnpm 10); the run selects Vitest directly. +- ttvc: `npm install --no-audit --no-fund`, `npm run build`, and + `node_modules/.bin/playwright install chromium`. The existing Playwright config + starts its application through `yarn express`. The npm preparation updated its + Yarn lockfile; the testdrive itself left the prepared dependency files unchanged. +- Cypress Kitchen Sink: `npm ci --no-audit --no-fund`, then `npm start` on port 8080. + Browser installation is part of project preparation, not a testdrive operation. +- CloudEvents: `git submodule update --init --depth 1`, `npm ci --no-audit --no-fund`, + `npm run build:schema`, then `npm run build:src`. Its conformance submodule was + `eddc279339609ed92d128bcd2b0d5c558a7ce396`. +- itsdangerous: create/activate a Python 3.11 venv and run + `python -m pip install -e . pytest freezegun`. +- concurrent-ruby and i18n: their original Gemfiles were evaluated by the isolated + bundle; no project bundle or manifest changes were needed. + +Run `ddtest onboard --framework FRAMEWORK` in each repository, followed by the +commands below. Package scripts used in the recorded runs are selected explicitly +with `--command`; automatic detection does not override framework commands. + +**typestack/class-validator** + +```sh +ddtest testdrive --framework jest --yes --command 'npm test -- --runInBand' +``` + +**omichelsen/compare-versions** + +```sh +ddtest testdrive --framework mocha --yes --command 'npm test' +``` + +**unjs/destr** + +```sh +ddtest testdrive --framework vitest --yes --command 'node_modules/.bin/vitest run' +``` + +**dropbox/ttvc** + +```sh +ddtest testdrive --framework playwright --yes --command 'node_modules/.bin/playwright test test/e2e/stylesheet1 --project=chromium --workers=1' +``` + +**cypress-io/cypress-example-kitchensink** + +```sh +ddtest testdrive --framework cypress --yes --command 'node_modules/.bin/cypress run --spec cypress/e2e/1-getting-started/todo.cy.js' +``` + +**cloudevents/sdk-javascript** + +```sh +ddtest testdrive --framework cucumber --yes --command 'npm run conformance' +``` + +**pallets/itsdangerous** + +```sh +ddtest testdrive --framework pytest --yes +``` + +**ruby-concurrency/concurrent-ruby** + +```sh +ddtest testdrive --framework rspec --yes --command 'bundle exec rspec spec/concurrent/atomic/atomic_boolean_spec.rb' +``` + +**ruby-i18n/i18n** + +```sh +ddtest testdrive --framework minitest --yes +``` + +Each run prints an absolute `Open report:` link. The report, `test-output.txt`, +and decoded `intake/*.json` files remain under its unique +`.testoptimization/testdrive//` directory. + +Run the reproducible fixture matrix from DDTest itself: + +```sh +GOTOOLCHAIN=go1.26.5 make test +GOTOOLCHAIN=go1.26.5 make lint +GOTOOLCHAIN=go1.26.5 \ + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1 \ + DDTEST_RUN_NPM_INTEGRATION_TEST=1 \ + go test ./internal/testdrive \ + -run 'TestPublicFrameworkTestdrives|TestInstrumentedJestFixture' \ + -count=1 -v +``` + +The integration matrix installs project dependencies in temporary directories, +including Cypress's browser. Python, pip/venv, Ruby, Bundler, Node, npm, and native +gem build tools must already be installed. + +## Problems found and addressed + +- Python's `uv run tox` and Ruby's `bundle exec rake` workflows were initially + missed. They are now included as candidate workflows for the coding agent to + inspect; this does not claim to statically validate every CI job. +- The pinned JavaScript tracer dereferenced a missing `scenario.id` for Cucumber + Background nodes when impacted-test detection was enabled. Local Cucumber runs + disable that feature, and generated CI guidance carries the same workaround. + CloudEvents' conformance scenarios and the independent + [Cucumber 7 TypeScript starter](https://github.com/hdorgeval/cucumber7-ts-starter/tree/02fb70d50ce0ccbad7ccc51fc856362f5efc5470) + passed after this change. The starter has no GitHub Actions workflow, so its + extra check covered testdrive only. A Background is now in the fixture matrix. +- Returning a completely empty known-tests map disabled Jest's existing retry + behavior. The intake now supplies empty datasets for all supported runner names; + the existing Jest retry and concurrent-session assertions remain intact. +- Cypress starts its config process beside the generated wrapper. The wrapper now + restores the project working directory before loading the original config, so + relative file access and existing after-run hooks keep their original behavior. + The fixture asserts both a customer task and an after-run file write. +- Ruby's native tracer extension failed to compile under a path containing spaces. + Testdrive now reports this prerequisite directly. Use a checkout without spaces + for Ruby; JavaScript and Python fixtures cover paths containing spaces. +- Ruby uses a copied lockfile with relocated local PATH sources, preserving the + customer's resolved versions where compatible while adding the pinned tracer + only to the session-owned bundle. + +## Deliberate limits + +Root projects and GitHub Actions onboarding only. No monorepo orchestration, +implicit application/browser installation, live Datadog connection validation, +public result schema, savings estimates, or Milestone 3/4 feature guarantees. +Only macOS arm64 was used for this open-source sample. Coverage depends on the +framework/tracer and the project's existing coverage setup; unsupported or absent +coverage is shown explicitly rather than inferred from passing tests. From e1ea68bbde2901f56e086fbb5359202deb48b610 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 23 Sep 2026 09:11:05 +0200 Subject: [PATCH 28/35] Use first GOPATH entry for installs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9deb7588..3a1d9f89 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X github.com/DataDog/ddtest/internal/buildinfo.Version=$(VERSION) GOBIN ?= $(shell go env GOBIN) ifeq ($(strip $(GOBIN)),) -GOBIN := $(shell go env GOPATH)/bin +GOBIN := $(firstword $(subst :, ,$(shell go env GOPATH)))/bin endif clean: go clean -i -x From 5bf8f434f75830fc735f637cf4d5e787dde59851 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 13:43:23 +0200 Subject: [PATCH 29/35] Document testdrive release and Git revision selection --- README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3d887f6e..0963e822 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,26 @@ Receiving events does not verify test skipping, EFD, or Test Management behavior Keep this directory out of source control. Each run has its own installation, files, and loopback port. +Testdrive installs the latest tracer by default. To choose a release or a Git +revision from the language's Datadog tracer repository: + +```sh +ddtest testdrive --tracer-version 6.15.0 --yes # JavaScript example +ddtest testdrive --tracer-version 'git:' --yes +``` + +The same option works for JavaScript, Python, and Ruby. Git selections also accept +branches and tags; they require Git and the tracer's source-build prerequisites. + Local testdrive prerequisites: -- JavaScript: Node.js 22+, npm for the isolated `dd-trace@6.15.0` installation, +- JavaScript: Node.js 22+, npm for the isolated `dd-trace` installation, and the project's package manager and dependencies. Vitest/ESM loading and Cypress config/support wrappers are supplied automatically. -- Python: an activated project environment with pytest and pip. `ddtrace==4.15.1` +- Python: an activated project environment with pytest and pip. `ddtrace` is installed into a session-owned directory; the active environment is unchanged. - Ruby: Ruby/Bundler and native gem build tools. A session-owned bundle adds - `datadog-ci@1.39.0`, preserving the original Gemfile and lockfile. The pinned + `datadog-ci`, preserving the original Gemfile and lockfile. The tracer's native extensions require a checkout path without spaces. - Browser suites: install the project's browsers and start any required services first, or use its existing test command that manages them. Testdrive does not From 42b9a0d2a7b4066880c575e09b37ec27bae0f97d Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 15:10:59 +0200 Subject: [PATCH 30/35] Document project tracer precedence and fallback version selection --- README.md | 23 +++++++++++++---------- docs/design/agent-driven-onboarding.md | 10 +++++----- docs/testing/onboarding-milestone-2.md | 4 ++++ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 0963e822..ea404b51 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,12 @@ separate instrumentation success from failed tests, and explicitly indicate when coverage was not reported. Tracer configuration errors are shown separately. The local intake supports agentless traffic; Agent/EVP routing is not supported. Receiving events does not verify test skipping, EFD, or Test Management behavior. -Keep this directory out of source control. Each run -has its own installation, files, and loopback port. +Keep this directory out of source control. Each run has its own files and loopback port. -Testdrive installs the latest tracer by default. To choose a release or a Git -revision from the language's Datadog tracer repository: +Testdrive always uses the project's tracer when one is present, for every language. +Only when no tracer is present does it install the latest release inside the session. +`--tracer-version` selects a release or Git revision for that fallback installation; +it never replaces an existing project tracer: ```sh ddtest testdrive --tracer-version 6.15.0 --yes # JavaScript example @@ -60,14 +61,16 @@ branches and tags; they require Git and the tracer's source-build prerequisites. Local testdrive prerequisites: -- JavaScript: Node.js 22+, npm for the isolated `dd-trace` installation, +- JavaScript: Node.js 22+, npm if `dd-trace` needs to be installed, and the project's package manager and dependencies. Vitest/ESM loading and Cypress config/support wrappers are supplied automatically. -- Python: an activated project environment with pytest and pip. `ddtrace` - is installed into a session-owned directory; the active environment is unchanged. -- Ruby: Ruby/Bundler and native gem build tools. A session-owned bundle adds - `datadog-ci`, preserving the original Gemfile and lockfile. The - tracer's native extensions require a checkout path without spaces. +- Python: an activated project environment with pytest, and pip if `ddtrace` is + absent. Fallback installation uses a session-owned directory; the active + environment is unchanged. +- Ruby: Ruby/Bundler. If `datadog-ci` is absent, a session-owned bundle adds it, + preserving the original Gemfile and lockfile. This fallback needs native gem + build tools and a checkout path without spaces. An existing tracer uses the + project's original bundle. - Browser suites: install the project's browsers and start any required services first, or use its existing test command that manages them. Testdrive does not install browsers or start applications on its own. diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md index 86cbfe1d..5a62cf02 100644 --- a/docs/design/agent-driven-onboarding.md +++ b/docs/design/agent-driven-onboarding.md @@ -43,7 +43,7 @@ The working preview is in draft PR #128 on `anmarchenko/agentic-onboarding-runbo ### Local testdrive - Each run has a unique directory and kernel-assigned loopback port, so sessions can run concurrently. -- Pinned tracers (`dd-trace@6.15.0`, `ddtrace==4.15.1`, and `datadog-ci@1.39.0`) are installed inside each session. Python keeps the active interpreter; Ruby uses an overlay bundle and a copy of the project lockfile. Project dependency files are untouched. +- Every language reuses the project tracer when present. Shared platform probes distinguish absence from detection failures. Only absent tracers are installed inside the session: latest by default, or a release/Git revision selected with `--tracer-version`. Python keeps the selected interpreter; Ruby creates an overlay bundle only for fallback installation. Project dependency files are untouched. - The local intake supports the endpoints exercised by that tracer and enables Test Optimization, coverage, Intelligent Test Runner, Early Flake Detection, Auto Test Retries, Impacted Tests, failed-test replay, and Test Management. - Test events and test- or suite-level coverage are decoded. Raw multipart or msgpack payloads are not retained; saved traffic is JSON only. - Complete test output is saved separately. @@ -57,19 +57,19 @@ The terminal and self-contained HTML report answer: Problem cards appear only when the answer is yes. Affected tests are always visible and expand to show attempts, timings, errors, retry information, source excerpts, and coverage. Flaky tests use `test.final_status` and are not also reported as failed. Covered files appear one per line and are paginated 50 at a time. Separate paginated tabs list all suites and tests. -The report also shows event and coverage counts, framework result, tracer version, isolated installation, saved JSON traffic, and test output. `testdrive` prints it as an absolute clickable `file://` link. +The report also shows event and coverage counts, framework result, project-tracer reuse or the fallback tracer selection, saved JSON traffic, and test output. `testdrive` prints it as an absolute clickable `file://` link. ### Code map - `internal/onboard/`: repository detection and embedded Markdown instructions. - `internal/testdrive/`: session lifecycle, preview, execution, terminal output, and HTML report. -- `internal/testdrive/tracer/`: isolated tracer installation behind the `Tracer` interface. +- `internal/testdrive/tracer/`: project-tracer reuse and fallback installation behind the `Tracer` interface. - `internal/testdrive/intake/`: local intake, JSON capture, event and coverage decoding, and findings. -- `internal/platform/` and `internal/framework/`: existing detection and test commands that future milestones should reuse. +- `internal/platform/` and `internal/framework/`: shared tracer probes, platform/framework detection, and test commands. ### Evidence and limits -- Integration tests run the public CLI with real pinned tracers against fixtures for all nine frameworks. Coverage is reported only when supplied by the tracer. +- Existing integration tests run the public CLI with real tracers against fixtures for all nine frameworks. Coverage is reported only when supplied by the tracer. - A concurrent integration test proves port, traffic, and file isolation. - Unit tests cover decoding, final status, flaky tests, both coverage granularities, medians, source excerpts, HTML rendering, and confirmation behavior. - Dogfooding on React Native Paper recognized 1,363 events and coverage for all 680 logical tests, including Early Flake Detection retries. diff --git a/docs/testing/onboarding-milestone-2.md b/docs/testing/onboarding-milestone-2.md index 074c3610..db14e322 100644 --- a/docs/testing/onboarding-milestone-2.md +++ b/docs/testing/onboarding-milestone-2.md @@ -10,6 +10,10 @@ compatibility study. Browser and RSpec checks used representative subsets as identified below. These results do not imply every version or configuration of each framework is compatible. +This historical record predates project-tracer reuse. Current testdrive always uses +a project tracer when present and installs latest (or `--tracer-version`) only as a +fallback. The pinned versions below describe the recorded runs. + ## Environment and checks - macOS arm64; Go 1.26.5; Node.js 24.14.0; Python 3.11; Ruby 3.4.7. From 2405ab99c3d758643ae4eedd43922fa956ef544b Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 15:41:18 +0200 Subject: [PATCH 31/35] Document Platform-owned tracer preparation --- docs/design/agent-driven-onboarding.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md index 5a62cf02..05f8af23 100644 --- a/docs/design/agent-driven-onboarding.md +++ b/docs/design/agent-driven-onboarding.md @@ -63,9 +63,8 @@ The report also shows event and coverage counts, framework result, project-trace - `internal/onboard/`: repository detection and embedded Markdown instructions. - `internal/testdrive/`: session lifecycle, preview, execution, terminal output, and HTML report. -- `internal/testdrive/tracer/`: project-tracer reuse and fallback installation behind the `Tracer` interface. - `internal/testdrive/intake/`: local intake, JSON capture, event and coverage decoding, and findings. -- `internal/platform/` and `internal/framework/`: shared tracer probes, platform/framework detection, and test commands. +- `internal/platform/` and `internal/framework/`: platform-owned tracer detection and installation, platform/framework detection, and test commands. ### Evidence and limits From ca0c9a117725effca4faf1bbad25a6429ba80a66 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 16:01:42 +0200 Subject: [PATCH 32/35] Document simple tracer checks and isolated fallback --- README.md | 6 +++--- docs/design/agent-driven-onboarding.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ea404b51..bb9649fa 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,10 @@ The local intake supports agentless traffic; Agent/EVP routing is not supported. Receiving events does not verify test skipping, EFD, or Test Management behavior. Keep this directory out of source control. Each run has its own files and loopback port. -Testdrive always uses the project's tracer when one is present, for every language. -Only when no tracer is present does it install the latest release inside the session. +Testdrive reuses the project's tracer when the platform's tracer check succeeds. +If the check fails, it attempts to install the latest release inside the session. `--tracer-version` selects a release or Git revision for that fallback installation; -it never replaces an existing project tracer: +project dependency files remain unchanged: ```sh ddtest testdrive --tracer-version 6.15.0 --yes # JavaScript example diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md index 05f8af23..7ee8b28d 100644 --- a/docs/design/agent-driven-onboarding.md +++ b/docs/design/agent-driven-onboarding.md @@ -43,7 +43,7 @@ The working preview is in draft PR #128 on `anmarchenko/agentic-onboarding-runbo ### Local testdrive - Each run has a unique directory and kernel-assigned loopback port, so sessions can run concurrently. -- Every language reuses the project tracer when present. Shared platform probes distinguish absence from detection failures. Only absent tracers are installed inside the session: latest by default, or a release/Git revision selected with `--tracer-version`. Python keeps the selected interpreter; Ruby creates an overlay bundle only for fallback installation. Project dependency files are untouched. +- Every language reuses the project tracer when its standard platform check succeeds. If the check fails, testdrive attempts an isolated installation inside the session: latest by default, or a release/Git revision selected with `--tracer-version`. Python keeps the selected interpreter; Ruby creates an overlay bundle only for fallback installation. Project dependency files are untouched. - The local intake supports the endpoints exercised by that tracer and enables Test Optimization, coverage, Intelligent Test Runner, Early Flake Detection, Auto Test Retries, Impacted Tests, failed-test replay, and Test Management. - Test events and test- or suite-level coverage are decoded. Raw multipart or msgpack payloads are not retained; saved traffic is JSON only. - Complete test output is saved separately. From 525238f4c2c3c7a9308c5b1c751e33dcb6717735 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 16:07:34 +0200 Subject: [PATCH 33/35] Scope Ruby build-path findings to the observed failure --- README.md | 2 +- docs/testing/onboarding-milestone-2.md | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bb9649fa..ee58f3c3 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Local testdrive prerequisites: environment is unchanged. - Ruby: Ruby/Bundler. If `datadog-ci` is absent, a session-owned bundle adds it, preserving the original Gemfile and lockfile. This fallback needs native gem - build tools and a checkout path without spaces. An existing tracer uses the + build tools. Bundler reports any native extension build failure. An existing tracer uses the project's original bundle. - Browser suites: install the project's browsers and start any required services first, or use its existing test command that manages them. Testdrive does not diff --git a/docs/testing/onboarding-milestone-2.md b/docs/testing/onboarding-milestone-2.md index db14e322..3ef651d5 100644 --- a/docs/testing/onboarding-milestone-2.md +++ b/docs/testing/onboarding-milestone-2.md @@ -170,9 +170,20 @@ gem build tools must already be installed. restores the project working directory before loading the original config, so relative file access and existing after-run hooks keep their original behavior. The fixture asserts both a customer task and an after-run file write. -- Ruby's native tracer extension failed to compile under a path containing spaces. - Testdrive now reports this prerequisite directly. Use a checkout without spaces - for Ruby; JavaScript and Python fixtures cover paths containing spaces. +- One Ruby fixture failed in a temporary directory named `project space`, not + in the DDTest checkout. The captured run used Ruby 3.4.7, Bundler 4.0.16, + `datadog-ci` 1.39.0, `datadog` 2.42.0, and `libdatadog` 40.0.0.2.0 on arm64 macOS. + The failing extension was `datadog-2.42.0/ext/libdatadog_api`. After + `ruby extconf.rb`, RubyGems ran + `make DESTDIR= sitearchdir=./.gem.20260922-66018-82y98z sitelibdir=./.gem.20260922-66018-82y98z`. + While compiling `crashtracker.c`, Clang reported `no such file or directory` + for an include path beginning `space/.testoptimization/testdrive/` and ending + `lib/pkgconfig/../../include`; make exited with `crashtracker.o Error 1`. + This suggests an incorrectly escaped build path. The log does not contain the + full compiler invocation, so the responsible escaping step was not established. + It does not establish a general Ruby path limitation or explain failures in + checkouts without spaces. Testdrive now attempts the build and preserves + Bundler's actual error instead of rejecting paths in advance. - Ruby uses a copied lockfile with relocated local PATH sources, preserving the customer's resolved versions where compatible while adding the pinned tracer only to the session-owned bundle. From 317fa71b21a19819851b2bd05ab7dce56f37ef2b Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 16:16:56 +0200 Subject: [PATCH 34/35] Document independent Ruby session dependency resolution --- README.md | 4 +++- docs/testing/onboarding-milestone-2.md | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ee58f3c3..a69c88f7 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,9 @@ Local testdrive prerequisites: absent. Fallback installation uses a session-owned directory; the active environment is unchanged. - Ruby: Ruby/Bundler. If `datadog-ci` is absent, a session-owned bundle adds it, - preserving the original Gemfile and lockfile. This fallback needs native gem + preserving the original Gemfile and lockfile. Bundler resolves a fresh session + lockfile from the Gemfile; versions pinned only in the project lockfile may differ. + This fallback needs native gem build tools. Bundler reports any native extension build failure. An existing tracer uses the project's original bundle. - Browser suites: install the project's browsers and start any required services diff --git a/docs/testing/onboarding-milestone-2.md b/docs/testing/onboarding-milestone-2.md index 3ef651d5..17d3d396 100644 --- a/docs/testing/onboarding-milestone-2.md +++ b/docs/testing/onboarding-milestone-2.md @@ -184,9 +184,10 @@ gem build tools must already be installed. It does not establish a general Ruby path limitation or explain failures in checkouts without spaces. Testdrive now attempts the build and preserves Bundler's actual error instead of rejecting paths in advance. -- Ruby uses a copied lockfile with relocated local PATH sources, preserving the - customer's resolved versions where compatible while adding the pinned tracer - only to the session-owned bundle. +- Ruby's fallback Gemfile evaluates the project's Gemfile and adds the selected + tracer. Bundler resolves a fresh session lockfile; DDTest does not read or copy + the project's lockfile. Project Gemfile constraints still apply, but versions + pinned only in the project lockfile can differ in the testdrive bundle. ## Deliberate limits From 82a47d411164ca043d2a3b0aa034fcd2a407b8f8 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 24 Sep 2026 17:02:20 +0200 Subject: [PATCH 35/35] Use the project Ruby bundle for testdrive --- README.md | 13 ++++++------- docs/design/agent-driven-onboarding.md | 6 +++--- docs/testing/onboarding-milestone-2.md | 16 +++++++++------- internal/testdrive/multiframework_test.go | 1 + 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a69c88f7..65b61a15 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ Keep this directory out of source control. Each run has its own files and loopba Testdrive reuses the project's tracer when the platform's tracer check succeeds. If the check fails, it attempts to install the latest release inside the session. `--tracer-version` selects a release or Git revision for that fallback installation; -project dependency files remain unchanged: +JavaScript and Python installations leave project dependency files unchanged; +Ruby uses `bundle add datadog-ci`, which updates the project Gemfile and lockfile: ```sh ddtest testdrive --tracer-version 6.15.0 --yes # JavaScript example @@ -67,12 +68,10 @@ Local testdrive prerequisites: - Python: an activated project environment with pytest, and pip if `ddtrace` is absent. Fallback installation uses a session-owned directory; the active environment is unchanged. -- Ruby: Ruby/Bundler. If `datadog-ci` is absent, a session-owned bundle adds it, - preserving the original Gemfile and lockfile. Bundler resolves a fresh session - lockfile from the Gemfile; versions pinned only in the project lockfile may differ. - This fallback needs native gem - build tools. Bundler reports any native extension build failure. An existing tracer uses the - project's original bundle. +- Ruby: Ruby/Bundler. If `datadog-ci` is unavailable, testdrive runs + `bundle add datadog-ci` in the project, honoring `BUNDLE_GEMFILE` and the + existing bundle settings. Bundler updates the project Gemfile and lockfile; + tests use that same bundle. Native gem builds need build tools. - Browser suites: install the project's browsers and start any required services first, or use its existing test command that manages them. Testdrive does not install browsers or start applications on its own. diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md index 7ee8b28d..9e013d7d 100644 --- a/docs/design/agent-driven-onboarding.md +++ b/docs/design/agent-driven-onboarding.md @@ -20,7 +20,7 @@ The local testdrive requires no Datadog account, API key, or Agent. Credentials - Keep the main flow to `ddtest onboard` and `ddtest testdrive`. - Detect the repository instead of asking setup questions. - Show commands and filesystem changes before running them. -- Keep tracer installation isolated from project manifests and lockfiles. +- Keep JavaScript and Python tracer installations isolated; disclose that Ruby fallback uses bundle add and updates project dependency files. - Explain what worked, what did not, and the next useful action. - Give the user clickable local and CI links. - Dogfood a thin end-to-end slice before adding abstractions. @@ -43,7 +43,7 @@ The working preview is in draft PR #128 on `anmarchenko/agentic-onboarding-runbo ### Local testdrive - Each run has a unique directory and kernel-assigned loopback port, so sessions can run concurrently. -- Every language reuses the project tracer when its standard platform check succeeds. If the check fails, testdrive attempts an isolated installation inside the session: latest by default, or a release/Git revision selected with `--tracer-version`. Python keeps the selected interpreter; Ruby creates an overlay bundle only for fallback installation. Project dependency files are untouched. +- Every language reuses the project tracer when its standard platform check succeeds. If it fails, install latest or a release/Git revision selected with `--tracer-version`. JavaScript and Python install inside the session; Python keeps the selected interpreter. Ruby runs `bundle add datadog-ci` and tests against the project bundle, updating its Gemfile and lockfile. - The local intake supports the endpoints exercised by that tracer and enables Test Optimization, coverage, Intelligent Test Runner, Early Flake Detection, Auto Test Retries, Impacted Tests, failed-test replay, and Test Management. - Test events and test- or suite-level coverage are decoded. Raw multipart or msgpack payloads are not retained; saved traffic is JSON only. - Complete test output is saved separately. @@ -123,7 +123,7 @@ Implemented in vertical slices: 1. Reuse each platform and framework's `Detect` method and test command. Remove Jest-specific names from the shared report. 2. Add the remaining JavaScript frameworks using the existing isolated `dd-trace` installation. 3. Add one pinned isolated `ddtrace` installation for pytest. -4. Add one pinned isolated Ruby tracer installation shared by RSpec and Minitest. +4. Use bundle add for Ruby tracer installation shared by RSpec and Minitest. 5. Add a tiny real-tracer fixture for every pair and dogfood at least one real repository per platform. For every pair, `onboard` finds the relevant GitHub Actions job and prints one small setup. `testdrive` previews and runs the detected framework's normal command (or the explicit `--command` entry point), treats received events as proof even when tests fail, reports missing coverage honestly, and preserves the same terminal and HTML experience where the tracer supplies the data. diff --git a/docs/testing/onboarding-milestone-2.md b/docs/testing/onboarding-milestone-2.md index 17d3d396..2edfc873 100644 --- a/docs/testing/onboarding-milestone-2.md +++ b/docs/testing/onboarding-milestone-2.md @@ -24,7 +24,9 @@ fallback. The pinned versions below describe the recorded runs. lockfiles, a Cucumber Background, and preserved Cypress task/after-run hooks. - Existing instrumented Jest and concurrent-session isolation checks pass. - Root manifests and lockfiles were hashed immediately before and after each - open-source run, including presence/absence. All nine snapshots were unchanged. + open-source run, including presence/absence. All nine snapshots were unchanged in those recorded runs. Ruby now uses + `bundle add` for fallback installation, which intentionally updates its Gemfile + and lockfile; the earlier Ruby isolation result no longer describes this behavior. Dependency installation and application builds happened before those snapshots. - No Datadog credentials or Agent were used. This validates local instrumentation and generated onboarding guidance, not a connection to a live Datadog account. @@ -69,8 +71,9 @@ Repository preparation used: `eddc279339609ed92d128bcd2b0d5c558a7ce396`. - itsdangerous: create/activate a Python 3.11 venv and run `python -m pip install -e . pytest freezegun`. -- concurrent-ruby and i18n: their original Gemfiles were evaluated by the isolated - bundle; no project bundle or manifest changes were needed. +- concurrent-ruby and i18n: those recorded runs used the former isolated bundle. + Current Ruby testdrives use the project bundle and add a missing tracer with + `bundle add datadog-ci`. Run `ddtest onboard --framework FRAMEWORK` in each repository, followed by the commands below. Package scripts used in the recorded runs are selected explicitly @@ -184,10 +187,9 @@ gem build tools must already be installed. It does not establish a general Ruby path limitation or explain failures in checkouts without spaces. Testdrive now attempts the build and preserves Bundler's actual error instead of rejecting paths in advance. -- Ruby's fallback Gemfile evaluates the project's Gemfile and adds the selected - tracer. Bundler resolves a fresh session lockfile; DDTest does not read or copy - the project's lockfile. Project Gemfile constraints still apply, but versions - pinned only in the project lockfile can differ in the testdrive bundle. +- Ruby fallback now invokes `bundle add datadog-ci` using the project's Bundler + settings. Bundler owns Gemfile/lockfile changes, dependency resolution and gem + installation; DDTest does not construct or copy a separate bundle. ## Deliberate limits diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index 9c291e18..d40f4c55 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -46,6 +46,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { run.projectTracer = "" run.projectTracer = "" run.projectTracer = "" + run.projectTracer = "" executor := &fakeTestdriveExecutor{} run.executor = executor run.startIntake = func(string) (localIntake, error) {