From 992f7d38879361fafad9c886cbb4dfac6d62186b Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 12:48:54 +0200 Subject: [PATCH 1/4] Preserve project tracer environments and add runtime regression tests --- .github/workflows/ci.yml | 14 ++ .../compatibility/project_environment_test.go | 123 ++++++++++++++++++ internal/platform/javascript.go | 3 +- internal/platform/javascript_test.go | 2 +- internal/platform/ruby.go | 3 +- internal/platform/ruby_test.go | 5 +- 6 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 internal/compatibility/project_environment_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 127f2a99..931f9c41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,19 @@ jobs: with: python-version: "3.12" + - name: Set up Node.js for project environment regression tests + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + + - name: Install Yarn for the Plug'n'Play regression fixture + run: npm install --global yarn@1.22.22 + + - name: Set up Ruby for project environment regression tests + uses: ruby/setup-ruby@762794c140bbeda0f1224786aa33b4b46783a6c1 # v1 + with: + ruby-version: "3.4" + - name: Cache Go modules uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: @@ -67,6 +80,7 @@ jobs: run: go test -race -coverpkg=./... -coverprofile=coverage.out -covermode=atomic ./... env: DD_ENV: ci + DDTEST_REQUIRE_PROJECT_RUNTIMES: "1" - name: Upload coverage report to Datadog if: ${{ env.DATADOG_API_KEY_CONFIGURED == 'true' }} diff --git a/internal/compatibility/project_environment_test.go b/internal/compatibility/project_environment_test.go new file mode 100644 index 00000000..1497b1f5 --- /dev/null +++ b/internal/compatibility/project_environment_test.go @@ -0,0 +1,123 @@ +package compatibility + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/settings" + "github.com/stretchr/testify/require" +) + +// These fixtures use real runtimes and package resolution, with no downloaded +// tracer or mocked command output. CI installs all four prerequisite commands. +func requireRuntime(t *testing.T, name string) { + t.Helper() + if _, err := exec.LookPath(name); err != nil { + if os.Getenv("DDTEST_REQUIRE_PROJECT_RUNTIMES") == "1" { + t.Fatalf("missing required regression-test runtime %s: %v", name, err) + } + t.Skipf("%s is required for the project environment regression test: %v", name, err) + } +} + +func runFixtureCommand(t *testing.T, ctx context.Context, name string, args ...string) { + t.Helper() + output, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + require.NoError(t, err, "%s: %s", name, output) +} + +func TestJavaScriptProjectEnvironment(t *testing.T) { + requireRuntime(t, "node") + requireRuntime(t, "yarn") + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + root := filepath.Join(t.TempDir(), "project with spaces") + writeFixture(t, root, "package.json", `{"name":"pnp-regression","private":true,"dependencies":{"dd-trace":"file:./tracer"}}`) + writeFixture(t, root, "tracer/package.json", `{"name":"dd-trace","version":"1.0.0"}`) + writeFixture(t, root, "tracer/ci/init.js", "module.exports = {};\n") + t.Chdir(root) + t.Setenv("NODE_OPTIONS", "") + t.Setenv("NODE_PATH", "") + runFixtureCommand(t, ctx, "yarn", "install", "--enable-pnp", "--offline", "--ignore-scripts", "--cache-folder", filepath.Join(root, "cache")) + require.NoDirExists(t, filepath.Join(root, "node_modules")) + loader := filepath.Join(root, ".pnp.js") // Yarn Classic's PnP loader. + require.FileExists(t, loader) + javascript := platform.NewJavaScript() + // Prove the fixture cannot pass through ordinary node_modules resolution. + _, err := javascript.DetectTracer(ctx, platform.TracerOptions{}) + require.ErrorContains(t, err, "Cannot find module 'dd-trace/ci/init'") + t.Setenv("NODE_OPTIONS", "--require "+strconv.Quote(loader)+" --max-old-space-size=256") + require.NoError(t, javascript.SanityCheck(ctx)) + path, err := javascript.DetectTracer(ctx, platform.TracerOptions{}) + require.NoError(t, err) + require.FileExists(t, path) + // Reusing the project must not fall back to a network installation. + installation, err := javascript.InstallTestdriveTracer(ctx, platform.TracerOptions{Directory: t.TempDir(), Version: "git:must-not-install"}) + require.NoError(t, err) + require.True(t, installation.Project) + require.Equal(t, path, installation.Path) +} + +func TestRubyProjectEnvironment(t *testing.T) { + requireRuntime(t, "bundle") + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + root := t.TempDir() + writeFixture(t, root, "tracer/datadog-ci.gemspec", `Gem::Specification.new do |s| + s.name = "datadog-ci" + s.version = "1.31.0" + s.summary = "Regression fixture" + s.authors = ["DDTest"] + s.files = [] +end +`) + writeFixture(t, root, "support/project_setup.rb", "PROJECT_TRACER_PATH = '../tracer'\n") + writeFixture(t, root, "config/Gemfile.test", "gem 'datadog-ci', path: PROJECT_TRACER_PATH\n") + t.Chdir(root) + t.Setenv("BUNDLE_GEMFILE", filepath.Join(root, "config", "Gemfile.test")) + t.Setenv("BUNDLE_USER_HOME", filepath.Join(root, "bundle-home")) + t.Setenv("RUBYOPT", "-I./support -rproject_setup") + runFixtureCommand(t, ctx, "bundle", "lock", "--local") + ruby := platform.NewRuby(settings.TestSkippingLevelTest) + t.Setenv("RUBYOPT", "") + _, err := ruby.DetectTracer(ctx, platform.TracerOptions{}) + require.ErrorContains(t, err, "PROJECT_TRACER_PATH") + t.Setenv("RUBYOPT", "-I./support -rproject_setup") + require.NoError(t, ruby.SanityCheck(ctx)) + installation, err := ruby.InstallTestdriveTracer(ctx, platform.TracerOptions{Directory: t.TempDir(), Version: "git:must-not-install"}) + require.NoError(t, err) + require.True(t, installation.Project) +} + +func TestPythonProjectEnvironment(t *testing.T) { + requireRuntime(t, "python") + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + root := t.TempDir() + // A private venv prevents a globally installed ddtrace from hiding a failure + // to preserve PYTHONPATH. Metadata alone suffices for the version probe. + runFixtureCommand(t, ctx, "python", "-m", "venv", "--without-pip", filepath.Join(root, "venv")) + writeFixture(t, root, "packages/ddtrace-4.11.0.dist-info/METADATA", "Metadata-Version: 2.1\nName: ddtrace\nVersion: 4.11.0\n") + t.Chdir(root) + bin := "bin" + if runtime.GOOS == "windows" { + bin = "Scripts" + } + t.Setenv("PATH", filepath.Join(root, "venv", bin)+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("PYTHONPATH", "") + python := platform.NewPython() + _, err := python.DetectTracer(ctx, platform.TracerOptions{Command: "python"}) + require.ErrorContains(t, err, "PackageNotFoundError") + t.Setenv("PYTHONPATH", filepath.Join(root, "packages")) + require.NoError(t, python.SanityCheck(ctx)) + version, err := python.DetectTracer(ctx, platform.TracerOptions{Command: "python"}) + require.NoError(t, err) + require.Equal(t, "4.11.0", version) +} diff --git a/internal/platform/javascript.go b/internal/platform/javascript.go index 58235e6b..93c3f3bd 100644 --- a/internal/platform/javascript.go +++ b/internal/platform/javascript.go @@ -249,7 +249,8 @@ func isDirectJavaScriptCommand(script string, names ...string) bool { // DetectTracer resolves the project's CI preload using Node's module resolution. func (j *JavaScript) DetectTracer(ctx context.Context, _ TracerOptions) (string, error) { - path, err := tracerProbe(ctx, j.executor, "node", []string{"-e", resolveJavaScriptModule, ddTraceCIInitModule}, map[string]string{"NODE_OPTIONS": ""}) + // Project resolution may depend on NODE_OPTIONS (for example Yarn PnP). + path, err := tracerProbe(ctx, j.executor, "node", []string{"-e", resolveJavaScriptModule, ddTraceCIInitModule}, nil) if err != nil { return "", fmt.Errorf("failed to resolve %s: %w", ddTraceCIInitModule, err) } diff --git a/internal/platform/javascript_test.go b/internal/platform/javascript_test.go index 377a406f..c8f2901e 100644 --- a/internal/platform/javascript_test.go +++ b/internal/platform/javascript_test.go @@ -704,7 +704,7 @@ func TestJavaScriptInstall(t *testing.T) { }, }, }, executor.commands) - require.Equal(t, []map[string]string{{"NODE_OPTIONS": ""}, {"NODE_OPTIONS": "", "NPM_CONFIG_GLOBAL": "false", "npm_config_global": "false"}, {"NODE_OPTIONS": "", "NPM_CONFIG_GLOBAL": "false", "npm_config_global": "false"}}, executor.envs) + require.Equal(t, []map[string]string{nil, {"NODE_OPTIONS": "", "NPM_CONFIG_GLOBAL": "false", "npm_config_global": "false"}, {"NODE_OPTIONS": "", "NPM_CONFIG_GLOBAL": "false", "npm_config_global": "false"}}, executor.envs) } func TestJavaScriptInstallReportsNPMError(t *testing.T) { diff --git a/internal/platform/ruby.go b/internal/platform/ruby.go index c5053021..30752b58 100644 --- a/internal/platform/ruby.go +++ b/internal/platform/ruby.go @@ -194,7 +194,8 @@ func parseBundlerInfoVersion(output, gemName string) (version.Version, error) { // DetectTracer reads the project tracer's bundle information. func (r *Ruby) DetectTracer(ctx context.Context, _ TracerOptions) (string, error) { - return tracerProbe(ctx, r.executor, "bundle", []string{"info", requiredGemName}, map[string]string{"RUBYOPT": ""}) + // Gemfiles may depend on load paths and project setup supplied by RUBYOPT. + return tracerProbe(ctx, r.executor, "bundle", []string{"info", requiredGemName}, nil) } func (r *Ruby) InstallTestdriveTracer(ctx context.Context, options TracerOptions) (TracerInstallation, error) { diff --git a/internal/platform/ruby_test.go b/internal/platform/ruby_test.go index adb63e69..7be0cd8e 100644 --- a/internal/platform/ruby_test.go +++ b/internal/platform/ruby_test.go @@ -572,9 +572,8 @@ func TestRubyTracerVersions(t *testing.T) { require.NoError(t, err) require.Equal(t, TracerInstallation{}, result) require.Equal(t, command{name: "bundle", args: tc.args}, executor.commands[1]) - for _, env := range executor.envs { - require.Equal(t, map[string]string{"RUBYOPT": ""}, env) // Inherit the project bundle settings. - } + require.Nil(t, executor.envs[0]) // Detect with the complete project environment. + require.Equal(t, map[string]string{"RUBYOPT": ""}, executor.envs[1]) // Clean preloads only for installation. entries, err := os.ReadDir(directory) require.NoError(t, err) require.Empty(t, entries) From 8dfbc9632a4127b4707879dd3c344a63bc6f9dbc Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 13:03:10 +0200 Subject: [PATCH 2/4] Isolate tracer probe results from runtime logs --- .../compatibility/project_environment_test.go | 22 ++++++++- internal/platform/javascript.go | 10 ++-- internal/platform/javascript_test.go | 14 ++++-- internal/platform/platform.go | 24 ++++++++-- internal/platform/platform_test.go | 48 +++++++++++++++++++ internal/platform/python.go | 4 +- internal/platform/python_test.go | 2 +- internal/platform/ruby.go | 10 +++- internal/platform/ruby_test.go | 2 +- internal/testdrive/testdrive_test.go | 2 +- 10 files changed, 119 insertions(+), 19 deletions(-) diff --git a/internal/compatibility/project_environment_test.go b/internal/compatibility/project_environment_test.go index 1497b1f5..d94d0cab 100644 --- a/internal/compatibility/project_environment_test.go +++ b/internal/compatibility/project_environment_test.go @@ -49,11 +49,18 @@ func TestJavaScriptProjectEnvironment(t *testing.T) { require.NoDirExists(t, filepath.Join(root, "node_modules")) loader := filepath.Join(root, ".pnp.js") // Yarn Classic's PnP loader. require.FileExists(t, loader) + writeFixture(t, root, "noisy-preload.cjs", `process.stdout.write('startup log without newline'); +console.error('startup stderr'); +process.on('exit', () => { + process.stdout.write('shutdown log without newline'); + console.error('shutdown stderr'); +}); +`) javascript := platform.NewJavaScript() // Prove the fixture cannot pass through ordinary node_modules resolution. _, err := javascript.DetectTracer(ctx, platform.TracerOptions{}) require.ErrorContains(t, err, "Cannot find module 'dd-trace/ci/init'") - t.Setenv("NODE_OPTIONS", "--require "+strconv.Quote(loader)+" --max-old-space-size=256") + t.Setenv("NODE_OPTIONS", "--require "+strconv.Quote(loader)+" --require "+strconv.Quote(filepath.Join(root, "noisy-preload.cjs"))+" --max-old-space-size=256") require.NoError(t, javascript.SanityCheck(ctx)) path, err := javascript.DetectTracer(ctx, platform.TracerOptions{}) require.NoError(t, err) @@ -63,6 +70,8 @@ func TestJavaScriptProjectEnvironment(t *testing.T) { require.NoError(t, err) require.True(t, installation.Project) require.Equal(t, path, installation.Path) + // Use the detected result as an actual preload; log-contaminated paths fail. + runFixtureCommand(t, ctx, "node", "--require", path, "-e", "require('dd-trace/ci/init')") } func TestRubyProjectEnvironment(t *testing.T) { @@ -78,7 +87,7 @@ func TestRubyProjectEnvironment(t *testing.T) { s.files = [] end `) - writeFixture(t, root, "support/project_setup.rb", "PROJECT_TRACER_PATH = '../tracer'\n") + writeFixture(t, root, "support/project_setup.rb", "puts 'startup log'\nwarn 'startup stderr'\nat_exit { puts 'shutdown log'; warn 'shutdown stderr' }\nPROJECT_TRACER_PATH = '../tracer'\n") writeFixture(t, root, "config/Gemfile.test", "gem 'datadog-ci', path: PROJECT_TRACER_PATH\n") t.Chdir(root) t.Setenv("BUNDLE_GEMFILE", filepath.Join(root, "config", "Gemfile.test")) @@ -91,6 +100,9 @@ end require.ErrorContains(t, err, "PROJECT_TRACER_PATH") t.Setenv("RUBYOPT", "-I./support -rproject_setup") require.NoError(t, ruby.SanityCheck(ctx)) + version, err := ruby.DetectTracer(ctx, platform.TracerOptions{}) + require.NoError(t, err) + require.Equal(t, " * datadog-ci (1.31.0)", version) installation, err := ruby.InstallTestdriveTracer(ctx, platform.TracerOptions{Directory: t.TempDir(), Version: "git:must-not-install"}) require.NoError(t, err) require.True(t, installation.Project) @@ -105,6 +117,12 @@ func TestPythonProjectEnvironment(t *testing.T) { // to preserve PYTHONPATH. Metadata alone suffices for the version probe. runFixtureCommand(t, ctx, "python", "-m", "venv", "--without-pip", filepath.Join(root, "venv")) writeFixture(t, root, "packages/ddtrace-4.11.0.dist-info/METADATA", "Metadata-Version: 2.1\nName: ddtrace\nVersion: 4.11.0\n") + writeFixture(t, root, "packages/sitecustomize.py", `import atexit, sys +print('startup log', end='') +print('startup stderr', file=sys.stderr) +atexit.register(lambda: print('shutdown log', end='')) +atexit.register(lambda: print('shutdown stderr', file=sys.stderr)) +`) t.Chdir(root) bin := "bin" if runtime.GOOS == "windows" { diff --git a/internal/platform/javascript.go b/internal/platform/javascript.go index 93c3f3bd..7a5fbfe1 100644 --- a/internal/platform/javascript.go +++ b/internal/platform/javascript.go @@ -254,10 +254,13 @@ func (j *JavaScript) DetectTracer(ctx context.Context, _ TracerOptions) (string, if err != nil { return "", fmt.Errorf("failed to resolve %s: %w", ddTraceCIInitModule, err) } + if !filepath.IsAbs(path) { + return "", fmt.Errorf("resolve %s: node returned non-absolute path %q", ddTraceCIInitModule, path) + } return path, nil } -const resolveJavaScriptModule = "process.stdout.write(require.resolve(process.argv[1]))" +const resolveJavaScriptModule = "require('fs').writeFileSync(process.argv[2], require.resolve(process.argv[1]))" // InstallTestdriveTracer reuses the project preload or installs an isolated fallback. func (j *JavaScript) InstallTestdriveTracer(ctx context.Context, options TracerOptions) (TracerInstallation, error) { @@ -276,12 +279,11 @@ func (j *JavaScript) InstallTestdriveTracer(ctx context.Context, options TracerO } ciInitModule := filepath.Join(sessionDirectory, "node_modules", "dd-trace", "ci", "init") - output, stderr, err := j.executor.Output(ctx, "node", []string{"-e", resolveJavaScriptModule, ciInitModule}, cleanEnvironment) + ciInitPath, err := tracerProbe(ctx, j.executor, "node", []string{"-e", resolveJavaScriptModule, ciInitModule}, cleanEnvironment) if err != nil { - return TracerInstallation{}, runtimeTagProbeError("resolve dd-trace/ci/init", stderr, err) + return TracerInstallation{}, fmt.Errorf("resolve dd-trace/ci/init: %w", err) } - ciInitPath := strings.TrimSpace(string(output)) if !filepath.IsAbs(ciInitPath) { return TracerInstallation{}, fmt.Errorf("resolve dd-trace/ci/init: node returned non-absolute path %q", ciInitPath) } diff --git a/internal/platform/javascript_test.go b/internal/platform/javascript_test.go index c8f2901e..1229c653 100644 --- a/internal/platform/javascript_test.go +++ b/internal/platform/javascript_test.go @@ -421,7 +421,7 @@ func TestJavaScript_SanityCheck_Passes(t *testing.T) { if calls == 1 && (len(args) != 1 || args[0] != "--version") { t.Fatalf("expected node --version, got %v", args) } - if calls == 2 && (len(args) != 3 || args[0] != "-e" || args[2] != ddTraceCIInitModule) { + if calls == 2 && (len(args) != 4 || args[0] != "-e" || args[2] != ddTraceCIInitModule) { t.Fatalf("expected node require.resolve command, got %v", args) } }, @@ -629,7 +629,7 @@ func (m *sequentialMockExecutor) Output(ctx context.Context, name string, args [ if err != nil { return nil, output, err } - return output, nil, nil + return output, nil, writeMockProbeResult(args, output) } type command struct { @@ -659,6 +659,9 @@ func (e *fakeCommandExecutor) CombinedOutput(_ context.Context, name string, arg func (e *fakeCommandExecutor) Output(ctx context.Context, name string, args []string, env map[string]string) ([]byte, []byte, error) { _, err := e.CombinedOutput(ctx, name, args, env) response := e.responses[len(e.commands)-1] + if err == nil { + err = writeMockProbeResult(args, response.output) + } return response.output, response.stderr, err } @@ -679,7 +682,7 @@ func TestJavaScriptInstall(t *testing.T) { require.False(t, ciInitPath.Project) require.True(t, filepath.IsAbs(ciInitPath.Path)) require.Equal(t, "node", executor.commands[0].name) - require.Equal(t, []string{"-e", resolveJavaScriptModule, ddTraceCIInitModule}, executor.commands[0].args) + require.Equal(t, []string{"-e", resolveJavaScriptModule, ddTraceCIInitModule}, executor.commands[0].args[:3]) require.Equal(t, []command{ executor.commands[0], { @@ -701,6 +704,7 @@ func TestJavaScriptInstall(t *testing.T) { "-e", resolveJavaScriptModule, filepath.Join(sessionDirectory, "node_modules", "dd-trace", "ci", "init"), + executor.commands[2].args[3], }, }, }, executor.commands) @@ -731,7 +735,7 @@ func TestJavaScriptInstallReportsResolveErrorWithoutOutput(t *testing.T) { javascript := &JavaScript{executor: executor} _, err := javascript.InstallTestdriveTracer(context.Background(), TracerOptions{Directory: t.TempDir()}) - require.ErrorContains(t, err, "resolve dd-trace/ci/init: exit status 1") + require.ErrorContains(t, err, "resolve dd-trace/ci/init: detect project tracer: exit status 1") } func TestJavaScriptInstallReportsResolveStderr(t *testing.T) { @@ -744,7 +748,7 @@ func TestJavaScriptInstallReportsResolveStderr(t *testing.T) { path, err := javascript.InstallTestdriveTracer(context.Background(), TracerOptions{Directory: t.TempDir()}) require.Empty(t, path) - require.ErrorContains(t, err, "resolve dd-trace/ci/init: Cannot find module dd-trace/ci/init") + require.ErrorContains(t, err, "resolve dd-trace/ci/init: detect project tracer: Cannot find module dd-trace/ci/init") require.ErrorIs(t, err, exitErr) } diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 6f06de35..1746d305 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -142,9 +142,27 @@ type commandExecutor interface { } func tracerProbe(ctx context.Context, executor commandExecutor, command string, args []string, env map[string]string) (string, error) { - output, stderr, err := executor.Output(ctx, command, args, env) + // Pass a private result file as the final argument. Runtime preloads and + // shutdown hooks can log to either output stream before or after the probe. + result, err := os.CreateTemp("", "ddtest-tracer-probe-*") if err != nil { - return "", runtimeTagProbeError("detect project tracer", stderr, err) + return "", fmt.Errorf("create tracer probe result: %w", err) } - return strings.TrimSpace(string(output)), nil + defer func() { _ = os.Remove(result.Name()) }() + if err := result.Close(); err != nil { + return "", fmt.Errorf("close tracer probe result: %w", err) + } + probeArgs := append(append([]string{}, args...), result.Name()) + stdout, stderr, err := executor.Output(ctx, command, probeArgs, env) + if err != nil { + return "", runtimeTagProbeError("detect project tracer", append(stdout, stderr...), err) + } + output, err := os.ReadFile(result.Name()) + if err != nil { + return "", fmt.Errorf("read tracer probe result: %w", err) + } + if len(output) == 0 { + return "", fmt.Errorf("tracer probe returned no result") + } + return string(output), nil } diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index 314ca3fe..6437a3c3 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "path/filepath" + "slices" "strings" "testing" @@ -16,6 +17,53 @@ import ( "github.com/stretchr/testify/require" ) +// Mock the probe scripts' result-file write while leaving stdout available for +// diagnostics, just like the real runtime executors. +func writeMockProbeResult(args []string, output []byte) error { + if slices.Contains(args, resolveJavaScriptModule) || slices.Contains(args, detectPythonTracer) { + return os.WriteFile(args[len(args)-1], []byte(strings.TrimSpace(string(output))), 0600) + } + return nil +} + +func TestTracerProbeSeparatesResultFromLogs(t *testing.T) { + for _, tc := range []struct { + name string + result string + processError error + wantError string + }{ + {name: "result", result: " /a path/with\na newline "}, + {name: "logs without result", wantError: "returned no result"}, + {name: "failure after result", result: "/valid-looking/path", processError: errors.New("process failed"), wantError: "process failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + var resultPath string + executor := &mockCommandExecutor{ + combinedOutput: []byte("startup log\nshutdown log without newline"), + combinedOutputErr: tc.processError, + onCombinedOutput: func(_ string, args []string, _ map[string]string) { + resultPath = args[len(args)-1] + require.NoError(t, os.WriteFile(resultPath, []byte(tc.result), 0600)) + }, + } + result, err := tracerProbe(t.Context(), executor, "runtime", []string{"probe"}, nil) + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + require.Empty(t, result) + if tc.processError != nil { + require.ErrorIs(t, err, tc.processError) + require.ErrorContains(t, err, "startup log") + } + } else { + require.NoError(t, err) + require.Equal(t, tc.result, result) + } + require.NoFileExists(t, resultPath) + }) + } +} + func TestPlatformsDetectTheirProjectFiles(t *testing.T) { tests := []struct { name string diff --git a/internal/platform/python.go b/internal/platform/python.go index 484bac3f..f61eb448 100644 --- a/internal/platform/python.go +++ b/internal/platform/python.go @@ -177,10 +177,12 @@ func (p *Python) SanityCheck(ctx context.Context) error { // DetectTracer returns the installed version using the test runner's interpreter. func (p *Python) DetectTracer(ctx context.Context, options TracerOptions) (string, error) { command, prefix := pythonInterpreter(options.Command, options.Args) - args := append(append([]string{}, prefix...), "-c", "import importlib.metadata, sys; print(importlib.metadata.version(sys.argv[1]))", requiredPackageName) + args := append(append([]string{}, prefix...), "-c", detectPythonTracer, requiredPackageName) return tracerProbe(ctx, p.executor, command, args, nil) } +const detectPythonTracer = "import importlib.metadata, pathlib, sys; pathlib.Path(sys.argv[2]).write_text(importlib.metadata.version(sys.argv[1]))" + func pythonInterpreter(command string, args []string) (string, []string) { base := strings.ToLower(filepath.Base(strings.ReplaceAll(command, `\`, "/"))) if isPythonExecutable(base) { diff --git a/internal/platform/python_test.go b/internal/platform/python_test.go index 33a70eb4..cf191fdf 100644 --- a/internal/platform/python_test.go +++ b/internal/platform/python_test.go @@ -489,7 +489,7 @@ func TestPythonTracerDetectionUsesSelectedEnvironment(t *testing.T) { executor := &mockCommandExecutor{combinedOutput: []byte("3.0.0\n"), onCombinedOutput: func(name string, args []string, env map[string]string) { require.Equal(t, "uv", name) require.Equal(t, []string{"run", "python", "-c"}, args[:3]) - require.Equal(t, "import importlib.metadata, sys; print(importlib.metadata.version(sys.argv[1]))", args[3]) + require.Equal(t, detectPythonTracer, args[3]) }} version, err := (&Python{executor: executor}).DetectTracer(context.Background(), TracerOptions{Command: "uv", Args: []string{"run", "python"}}) require.NoError(t, err) diff --git a/internal/platform/ruby.go b/internal/platform/ruby.go index 30752b58..408aa077 100644 --- a/internal/platform/ruby.go +++ b/internal/platform/ruby.go @@ -195,7 +195,15 @@ func parseBundlerInfoVersion(output, gemName string) (version.Version, error) { // DetectTracer reads the project tracer's bundle information. func (r *Ruby) DetectTracer(ctx context.Context, _ TracerOptions) (string, error) { // Gemfiles may depend on load paths and project setup supplied by RUBYOPT. - return tracerProbe(ctx, r.executor, "bundle", []string{"info", requiredGemName}, nil) + stdout, stderr, err := r.executor.Output(ctx, "bundle", []string{"info", requiredGemName}, nil) + if err != nil { + return "", runtimeTagProbeError("detect project tracer", append(stdout, stderr...), err) + } + gemVersion, err := parseBundlerInfoVersion(string(stdout), requiredGemName) + if err != nil { + return "", err + } + return fmt.Sprintf(" * %s (%s)", requiredGemName, gemVersion.String()), nil } func (r *Ruby) InstallTestdriveTracer(ctx context.Context, options TracerOptions) (TracerInstallation, error) { diff --git a/internal/platform/ruby_test.go b/internal/platform/ruby_test.go index 7be0cd8e..075f08f7 100644 --- a/internal/platform/ruby_test.go +++ b/internal/platform/ruby_test.go @@ -549,7 +549,7 @@ func (m *mockCommandExecutor) Output(ctx context.Context, name string, args []st if err != nil { return nil, output, err } - return output, nil, nil + return output, nil, writeMockProbeResult(args, output) } func TestRubyTracerVersions(t *testing.T) { diff --git a/internal/testdrive/testdrive_test.go b/internal/testdrive/testdrive_test.go index c5079cc0..e956e1b3 100644 --- a/internal/testdrive/testdrive_test.go +++ b/internal/testdrive/testdrive_test.go @@ -520,7 +520,7 @@ func TestPreviewChoosesTracerBeforeConfirmation(t *testing.T) { bin := t.TempDir() script := "#!/bin/sh\nexit 1\n" if installed { - script = "#!/bin/sh\nprintf /project/node_modules/dd-trace/ci/init.js\n" + script = "#!/bin/sh\nprintf /project/node_modules/dd-trace/ci/init.js > \"$4\"\n" } requireWriteFile(t, filepath.Join(bin, "node"), script) if err := os.Chmod(filepath.Join(bin, "node"), 0755); err != nil { From f2cda611b789197ca7d8884fe6c63c145eee568f Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 13:10:01 +0200 Subject: [PATCH 3/4] Load project Node options before tracer instrumentation --- docs/running.md | 13 ++--- .../compatibility/project_environment_test.go | 48 +++++++++++++++++-- internal/platform/javascript.go | 5 +- internal/platform/javascript_test.go | 2 +- internal/testdrive/javascript.go | 4 +- internal/testdrive/testdrive_test.go | 2 +- 6 files changed, 60 insertions(+), 14 deletions(-) diff --git a/docs/running.md b/docs/running.md index 41c9735f..6556fc3d 100644 --- a/docs/running.md +++ b/docs/running.md @@ -263,8 +263,9 @@ When `--tests-location` or `--tests-exclude-pattern` is set, DDTest filters the file list returned by Jest after discovery; it does not pass `--tests-location` as Jest's `--testMatch`. -DDTest prepends `-r dd-trace/ci/init` to `NODE_OPTIONS` for worker processes -unless `NODE_OPTIONS` already loads `dd-trace/ci/init`. +DDTest appends `-r dd-trace/ci/init` to `NODE_OPTIONS` for worker processes +unless `NODE_OPTIONS` already loads `dd-trace/ci/init`. Existing project loaders, +such as Yarn Plug'n'Play, run before the tracer. ## Cucumber Discovery And Instrumentation @@ -336,10 +337,10 @@ projects, include and exclude patterns, and CLI filters without executing tests. If that API is unavailable, DDTest falls back to its own filesystem glob using `--tests-location` or the default Vitest test-file pattern. -DDTest prepends both `--import dd-trace/register.js` and -`-r dd-trace/ci/init` to `NODE_OPTIONS` for Vitest worker processes unless they -are already present. Discovery removes these options to avoid instrumenting the -file-listing process. +DDTest adds `--import dd-trace/register.js` and `-r dd-trace/ci/init` to +`NODE_OPTIONS` for Vitest worker processes unless they are already present. +The tracer's `--require` option follows existing project loaders. Discovery +removes these options to avoid instrumenting the file-listing process. ## Cypress Discovery And Instrumentation diff --git a/internal/compatibility/project_environment_test.go b/internal/compatibility/project_environment_test.go index d94d0cab..f77df805 100644 --- a/internal/compatibility/project_environment_test.go +++ b/internal/compatibility/project_environment_test.go @@ -1,6 +1,7 @@ package compatibility import ( + "bytes" "context" "os" "os/exec" @@ -10,8 +11,10 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/settings" + "github.com/DataDog/ddtest/internal/testdrive" "github.com/stretchr/testify/require" ) @@ -39,9 +42,11 @@ func TestJavaScriptProjectEnvironment(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), time.Minute) defer cancel() root := filepath.Join(t.TempDir(), "project with spaces") - writeFixture(t, root, "package.json", `{"name":"pnp-regression","private":true,"dependencies":{"dd-trace":"file:./tracer"}}`) - writeFixture(t, root, "tracer/package.json", `{"name":"dd-trace","version":"1.0.0"}`) - writeFixture(t, root, "tracer/ci/init.js", "module.exports = {};\n") + writeFixture(t, root, "package.json", `{"name":"pnp-regression","private":true,"scripts":{"test":"jest"},"dependencies":{"dd-trace":"file:./tracer"}}`) + writeFixture(t, root, "tracer/package.json", `{"name":"dd-trace","version":"1.0.0","dependencies":{"pnp-tracer-helper":"file:../helper"}}`) + writeFixture(t, root, "helper/package.json", `{"name":"pnp-tracer-helper","version":"1.0.0","main":"index.js"}`) + writeFixture(t, root, "helper/index.js", "module.exports = 'loaded through PnP';\n") + writeFixture(t, root, "tracer/ci/init.js", "global.ddtestTracer = require('pnp-tracer-helper');\n") t.Chdir(root) t.Setenv("NODE_OPTIONS", "") t.Setenv("NODE_PATH", "") @@ -72,6 +77,43 @@ process.on('exit', () => { require.Equal(t, path, installation.Path) // Use the detected result as an actual preload; log-contaminated paths fail. runFixtureCommand(t, ctx, "node", "--require", path, "-e", "require('dd-trace/ci/init')") + + // Exercise the actual discovery/run adapters and testdrive worker startup. + // The tracer's dependency also needs PnP when the tracer path is absolute. + resetSettingsAfterTest(t) + writeFixture(t, root, "example.test.js", "// Worker-startup fixture.\n") + writeFixture(t, root, "worker.cjs", `const assert = require('assert'); +const fs = require('fs'); +if (process.argv.includes('--listTests')) { + assert.strictEqual(global.ddtestTracer, undefined); + console.log(require('path').resolve('example.test.js')); +} else { + assert.strictEqual(global.ddtestTracer, 'loaded through PnP'); + fs.writeFileSync('worker-ran', 'instrumented'); +} +`) + configureFramework(shellCommand("node", filepath.Join(root, "worker.cjs")), "") + t.Setenv("NODE_OPTIONS", "--require "+strconv.Quote(loader)+" --max-old-space-size=256") + t.Run("discovery and execution", func(t *testing.T) { + fw, err := javascript.DetectFramework() + require.NoError(t, err) + files, err := fw.DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: "**/*.test.js"}) + require.NoError(t, err) + require.Len(t, files, 1) + require.NoError(t, fw.RunTests(ctx, files, nil)) + require.FileExists(t, filepath.Join(root, "worker-ran")) + require.NoError(t, os.Remove(filepath.Join(root, "worker-ran"))) + }) + t.Run("testdrive", func(t *testing.T) { + drive, err := testdrive.Prepare("git:must-not-install") + require.NoError(t, err) + var output bytes.Buffer + err = drive.Run(ctx, &output) + // This fixture checks startup, not telemetry. A successful command with + // no events is distinguishable from a failed Node preload. + require.ErrorContains(t, err, "command exited successfully, but Test Optimization sent no test events", output.String()) + require.FileExists(t, filepath.Join(root, "worker-ran")) + }) } func TestRubyProjectEnvironment(t *testing.T) { diff --git a/internal/platform/javascript.go b/internal/platform/javascript.go index 7a5fbfe1..c37a3e51 100644 --- a/internal/platform/javascript.go +++ b/internal/platform/javascript.go @@ -97,10 +97,11 @@ func (j *JavaScript) GetPlatformEnv() map[string]string { return map[string]string{} } - // Keep user-provided options after the required Datadog preloads. + // Project loaders (for example Yarn PnP) must run before the tracer can + // resolve itself and its dependencies. Preserve their existing order. nodeOptions := nodeOptionsDDTraceCIArg if strings.TrimSpace(currentValue) != "" { - nodeOptions += " " + currentValue + nodeOptions = currentValue + " " + nodeOptions } slog.Debug("Setting NODE_OPTIONS to auto-instrument with dd-trace-js", "nodeOptions", nodeOptions) diff --git a/internal/platform/javascript_test.go b/internal/platform/javascript_test.go index 1229c653..38e010c9 100644 --- a/internal/platform/javascript_test.go +++ b/internal/platform/javascript_test.go @@ -70,7 +70,7 @@ func TestJavaScript_GetPlatformEnv_PreservesExistingNODEOPTIONS(t *testing.T) { javascript := NewJavaScript() envMap := javascript.GetPlatformEnv() - expected := nodeOptionsDDTraceCIArg + " --max-old-space-size=4096" + expected := "--max-old-space-size=4096 " + nodeOptionsDDTraceCIArg if envMap[nodeOptionsEnvVar] != expected { t.Errorf("expected NODE_OPTIONS to be %q, got %q", expected, envMap[nodeOptionsEnvVar]) } diff --git a/internal/testdrive/javascript.go b/internal/testdrive/javascript.go index 0ec32f77..ccc2b159 100644 --- a/internal/testdrive/javascript.go +++ b/internal/testdrive/javascript.go @@ -11,7 +11,9 @@ import ( func javascriptEnvironment(ciInitPath string) map[string]string { nodeOptions := "-r " + strconv.Quote(ciInitPath) if current := stripDatadogNodeOptions(os.Getenv("NODE_OPTIONS")); current != "" { - nodeOptions += " " + current + // Even an absolute tracer path can depend on the project's loader to + // resolve its dependencies. + nodeOptions = current + " " + nodeOptions } return map[string]string{"NODE_OPTIONS": nodeOptions} diff --git a/internal/testdrive/testdrive_test.go b/internal/testdrive/testdrive_test.go index e956e1b3..6be86cf5 100644 --- a/internal/testdrive/testdrive_test.go +++ b/internal/testdrive/testdrive_test.go @@ -421,7 +421,7 @@ func TestRunReportsTestOutputWriteFailure(t *testing.T) { func TestTestEnvironmentPreservesExistingNodeOptions(t *testing.T) { t.Setenv("NODE_OPTIONS", "--require dd-trace/ci/init --max-old-space-size=4096 --import=/tmp/dd-trace/register.js") environment := javascriptEnvironment("/tmp/dd-trace/ci/init.js") - if environment["NODE_OPTIONS"] != `-r "/tmp/dd-trace/ci/init.js" --max-old-space-size=4096` { + if environment["NODE_OPTIONS"] != `--max-old-space-size=4096 -r "/tmp/dd-trace/ci/init.js"` { t.Fatalf("NODE_OPTIONS = %q", environment["NODE_OPTIONS"]) } } From d6292f4b6d75b23f2cfe74cee4952f35fb596fc3 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 28 Sep 2026 13:54:33 +0200 Subject: [PATCH 4/4] Protect Jest discovery from logs and fix local testdrive startup --- .github/workflows/ci.yml | 4 +- internal/compatibility/jest_test.go | 38 +++++++++ .../compatibility/project_environment_test.go | 4 +- internal/framework/jest.go | 53 ++++++++++--- internal/framework/jest_test.go | 77 +++++++++++++++---- internal/testdrive/intake/settings.go | 15 ++-- internal/testdrive/intake/settings_test.go | 2 +- 7 files changed, 154 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 931f9c41..0bc682f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,9 +241,9 @@ jobs: - name: Test Jest adapter run: | jest_dir="${RUNNER_TEMP}/jest-${{ matrix.jest }}" - npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" + npm install --prefix "${jest_dir}" "jest@${{ matrix.jest }}" dd-trace@5.86.0 DDTEST_JEST_NODE_MODULES="${jest_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestJestAdapterIntegration$' + go test -v ./internal/compatibility -run '^TestJest(Adapter|Testdrive)Integration$' vitest-compatibility: runs-on: ubuntu-latest diff --git a/internal/compatibility/jest_test.go b/internal/compatibility/jest_test.go index 013a2a58..54ed2300 100644 --- a/internal/compatibility/jest_test.go +++ b/internal/compatibility/jest_test.go @@ -1,14 +1,18 @@ package compatibility import ( + "bytes" "context" "os" "path/filepath" + "strconv" "testing" "time" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/testdrive" + "github.com/stretchr/testify/require" ) func TestJestAdapterIntegration(t *testing.T) { @@ -33,6 +37,14 @@ func TestJestAdapterIntegration(t *testing.T) { throw new Error('unselected file ran') }) `) + writeFixture(t, root, "noisy.cjs", `process.stdout.write('startup without newline'); +process.stderr.write('startup stderr'); +process.on('exit', () => { + process.stdout.write('shutdown without newline'); + process.stderr.write('shutdown stderr'); +}); +`) + t.Setenv("NODE_OPTIONS", "--require "+strconv.Quote(filepath.Join(root, "noisy.cjs"))) t.Chdir(root) jest := framework.NewJest() @@ -51,3 +63,29 @@ func TestJestAdapterIntegration(t *testing.T) { t.Fatalf("selected-file run failed: %v", err) } } + +// A real tracer must emit events, not merely exit successfully. In particular, +// dd-trace 5.86 waits forever for disabled git upload if local settings advertise +// test skipping, allowing Node to exit before Jest starts. +func TestJestTestdriveIntegration(t *testing.T) { + nodeModules := requireEnv(t, "DDTEST_JEST_NODE_MODULES") + root := t.TempDir() + require.NoError(t, os.Symlink(nodeModules, filepath.Join(root, "node_modules"))) + writeFixture(t, root, "package.json", `{"name":"testdrive-regression","private":true,"scripts":{"test":"jest"}}`) + writeFixture(t, root, "example.test.js", `test('runs the test body', () => { expect(1 + 1).toBe(2); });`) + writeFixture(t, root, "noisy.cjs", `process.stdout.write('startup'); process.on('exit', () => process.stdout.write('shutdown'));`) + t.Chdir(root) + t.Setenv("NODE_OPTIONS", "--require "+strconv.Quote(filepath.Join(root, "noisy.cjs"))) + resetSettingsAfterTest(t) + configureFramework(shellCommand("node", filepath.Join(root, "node_modules", "jest", "bin", "jest.js"), "--runInBand"), "") + drive, err := testdrive.Prepare("git:must-not-install") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + var output bytes.Buffer + require.NoError(t, drive.Run(ctx, &output), output.String()) + // Local EFD settings execute this one new test twice. + require.Contains(t, output.String(), "Test events: 2") + require.Contains(t, output.String(), "Jest: Passed") + require.Contains(t, output.String(), "reused") +} diff --git a/internal/compatibility/project_environment_test.go b/internal/compatibility/project_environment_test.go index f77df805..39e40f30 100644 --- a/internal/compatibility/project_environment_test.go +++ b/internal/compatibility/project_environment_test.go @@ -86,14 +86,14 @@ process.on('exit', () => { const fs = require('fs'); if (process.argv.includes('--listTests')) { assert.strictEqual(global.ddtestTracer, undefined); - console.log(require('path').resolve('example.test.js')); + assert(process.argv.includes('--json')); + console.log(JSON.stringify([require('path').resolve('example.test.js')])); } else { assert.strictEqual(global.ddtestTracer, 'loaded through PnP'); fs.writeFileSync('worker-ran', 'instrumented'); } `) configureFramework(shellCommand("node", filepath.Join(root, "worker.cjs")), "") - t.Setenv("NODE_OPTIONS", "--require "+strconv.Quote(loader)+" --max-old-space-size=256") t.Run("discovery and execution", func(t *testing.T) { fw, err := javascript.DetectFramework() require.NoError(t, err) diff --git a/internal/framework/jest.go b/internal/framework/jest.go index e853f79e..1d1dca19 100644 --- a/internal/framework/jest.go +++ b/internal/framework/jest.go @@ -1,7 +1,9 @@ package framework import ( + "bytes" "context" + "encoding/json" "errors" "fmt" "log/slog" @@ -99,7 +101,7 @@ func (j *Jest) DiscoverTestFiles(ctx context.Context, testFiles discovery.TestFi command, baseArgs := j.Command() args := slices.Clone(baseArgs) - args = withFrameworkOptions(command, args, "jest", "--listTests") + args = withFrameworkOptions(command, args, "jest", "--listTests", "--json") slog.Info("Discovering Jest test files with command", "command", command, "args", args) output, err := j.executor.CombinedOutput(ctx, command, args, j.discoveryEnv()) @@ -111,7 +113,10 @@ func (j *Jest) DiscoverTestFiles(ctx context.Context, testFiles discovery.TestFi return nil, fmt.Errorf("failed to discover Jest test files: %s: %w", message, err) } - discoveredFiles := parseJestListTestsOutput(output) + discoveredFiles, err := parseJestListTestsOutput(output) + if err != nil { + return nil, fmt.Errorf("failed to discover Jest test files: %w", err) + } if settings.GetTestsLocation() == "" && settings.GetTestsExcludePattern() == "" { return discoveredFiles, nil } @@ -221,18 +226,44 @@ func stripNodeOptionsRequire(nodeOptions string, module string) string { return strings.Join(stripped, " ") } -func parseJestListTestsOutput(output []byte) []string { +// Jest's --listTests --json writes an array of absolute paths. Preloads and +// package managers may log before or after it, even without a newline. Accept +// exactly one such array; missing or ambiguous output must not become an empty +// successful plan. +func parseJestListTestsOutput(output []byte) ([]string, error) { + var paths []string + found := false + for len(output) > 0 { + start := bytes.IndexByte(output, '[') + if start < 0 { + break + } + output = output[start:] + decoder := json.NewDecoder(bytes.NewReader(output)) + var candidate []string + if err := decoder.Decode(&candidate); err != nil { + output = output[1:] + continue + } + output = output[decoder.InputOffset():] + if slices.ContainsFunc(candidate, func(path string) bool { return !filepath.IsAbs(path) }) { + continue + } + if found { + return nil, errors.New("ambiguous Jest JSON test list") + } + paths, found = candidate, true + } + if !found { + return nil, errors.New("missing Jest JSON test list") + } + cwd, _ := os.Getwd() if resolvedCwd, err := filepath.EvalSymlinks(cwd); err == nil { cwd = resolvedCwd } testFiles := make([]string, 0) - for _, line := range strings.Split(string(output), "\n") { - testFile := strings.TrimSpace(line) - if testFile == "" { - continue - } - + for _, testFile := range paths { if filepath.IsAbs(testFile) && cwd != "" { pathForRel := testFile if resolvedPath, err := filepath.EvalSymlinks(testFile); err == nil { @@ -250,11 +281,11 @@ func parseJestListTestsOutput(output []byte) []string { continue } if _, err := os.Stat(normalizedTestFile); err != nil { - continue + return nil, fmt.Errorf("invalid Jest test file %q: %w", testFile, err) } testFiles = append(testFiles, normalizedTestFile) } slices.Sort(testFiles) - return slices.Compact(testFiles) + return slices.Compact(testFiles), nil } diff --git a/internal/framework/jest_test.go b/internal/framework/jest_test.go index 414ec4f3..78c8128d 100644 --- a/internal/framework/jest_test.go +++ b/internal/framework/jest_test.go @@ -2,6 +2,7 @@ package framework import ( "context" + "encoding/json" "errors" "os" "path/filepath" @@ -133,9 +134,7 @@ func TestJest_DiscoverTestFiles_UsesLocalJestListTests(t *testing.T) { var capturedName string var capturedArgs []string mockExecutor := &jestCommandExecutor{ - output: []byte(filepath.Join(tempDir, "src", "b.test.ts") + "\n" + - filepath.Join(tempDir, "src", "foo.test.js") + "\n" + - "warning: ignored because it is not a file\n"), + output: jestListOutput(filepath.Join(tempDir, "src", "b.test.ts"), filepath.Join(tempDir, "src", "foo.test.js")), onExecution: func(name string, args []string) { capturedName = name capturedArgs = slices.Clone(args) @@ -153,7 +152,7 @@ func TestJest_DiscoverTestFiles_UsesLocalJestListTests(t *testing.T) { if capturedName != binJestPath { t.Errorf("expected command %q, got %q", binJestPath, capturedName) } - expectedArgs := []string{"--listTests"} + expectedArgs := []string{"--listTests", "--json"} if !slices.Equal(capturedArgs, expectedArgs) { t.Errorf("expected args %v, got %v", expectedArgs, capturedArgs) } @@ -187,7 +186,7 @@ func TestJest_DiscoverTestFiles_StripsInheritedNodeOptions(t *testing.T) { } mockExecutor := &jestCommandExecutor{ - output: []byte(filepath.Join(tempDir, "src", "a.test.js") + "\n"), + output: jestListOutput(filepath.Join(tempDir, "src", "a.test.js")), } jest := &Jest{executor: mockExecutor, platformEnv: make(map[string]string)} @@ -231,9 +230,7 @@ func TestJest_DiscoverTestFiles_WithTestsLocationFiltersListTestsOutput(t *testi var capturedName string var capturedArgs []string mockExecutor := &jestCommandExecutor{ - output: []byte(filepath.Join(tempDir, "custom", "unit", "b.check.js") + "\n" + - filepath.Join(tempDir, "src", "c.test.js") + "\n" + - filepath.Join(tempDir, "custom", "unit", "a.check.js") + "\n"), + output: jestListOutput(filepath.Join(tempDir, "custom", "unit", "b.check.js"), filepath.Join(tempDir, "src", "c.test.js"), filepath.Join(tempDir, "custom", "unit", "a.check.js")), onExecution: func(name string, args []string) { capturedName = name capturedArgs = slices.Clone(args) @@ -248,7 +245,7 @@ func TestJest_DiscoverTestFiles_WithTestsLocationFiltersListTestsOutput(t *testi if capturedName != "npx" { t.Errorf("expected command %q, got %q", "npx", capturedName) } - expectedArgs := []string{"jest", "--listTests"} + expectedArgs := []string{"jest", "--listTests", "--json"} if !slices.Equal(capturedArgs, expectedArgs) { t.Errorf("expected args %v, got %v", expectedArgs, capturedArgs) } @@ -263,7 +260,7 @@ func TestJest_DiscoverTestFiles_WithTestsLocationReturnsInvalidPatternError(t *t setTestsLocation(t, "custom/[") mockExecutor := &jestCommandExecutor{ - output: []byte("custom/a.check.js\n"), + output: []byte("[]"), } jest := &Jest{executor: mockExecutor, platformEnv: make(map[string]string)} @@ -297,8 +294,7 @@ func TestJest_DiscoverTestFiles_WithTestsExcludePatternFiltersListTestsOutput(t var capturedArgs []string mockExecutor := &jestCommandExecutor{ - output: []byte(filepath.Join(tempDir, "src", "system", "b.test.js") + "\n" + - filepath.Join(tempDir, "src", "a.test.js") + "\n"), + output: jestListOutput(filepath.Join(tempDir, "src", "system", "b.test.js"), filepath.Join(tempDir, "src", "a.test.js")), onExecution: func(name string, args []string) { capturedArgs = slices.Clone(args) }, @@ -309,7 +305,7 @@ func TestJest_DiscoverTestFiles_WithTestsExcludePatternFiltersListTestsOutput(t t.Fatalf("DiscoverTestFiles failed: %v", err) } - expectedArgs := []string{"jest", "--listTests"} + expectedArgs := []string{"jest", "--listTests", "--json"} if !slices.Equal(capturedArgs, expectedArgs) { t.Errorf("expected args %v, got %v", expectedArgs, capturedArgs) } @@ -337,7 +333,7 @@ func TestJest_DiscoverTestFiles_WithOverride(t *testing.T) { var capturedName string var capturedArgs []string mockExecutor := &jestCommandExecutor{ - output: []byte(filepath.Join(tempDir, "src", "a.test.js") + "\n"), + output: jestListOutput(filepath.Join(tempDir, "src", "a.test.js")), onExecution: func(name string, args []string) { capturedName = name capturedArgs = slices.Clone(args) @@ -356,7 +352,7 @@ func TestJest_DiscoverTestFiles_WithOverride(t *testing.T) { if capturedName != "pnpm" { t.Errorf("expected command %q, got %q", "pnpm", capturedName) } - expectedArgs := []string{"jest", "--runInBand", "--listTests"} + expectedArgs := []string{"jest", "--runInBand", "--listTests", "--json"} if !slices.Equal(capturedArgs, expectedArgs) { t.Errorf("expected args %v, got %v", expectedArgs, capturedArgs) } @@ -508,7 +504,7 @@ func TestJestSeparatorPreservesOptionsAndReplacesSelection(t *testing.T) { {"npx", "--", "jest", "--runInBand", "--", "old.test.js"}, } { var got []string - j := &Jest{commandOverride: override, executor: &jestCommandExecutor{onExecution: func(_ string, args []string) { got = slices.Clone(args) }}} + j := &Jest{commandOverride: override, executor: &jestCommandExecutor{output: []byte("[]"), onExecution: func(_ string, args []string) { got = slices.Clone(args) }}} if err := j.RunTests(t.Context(), []string{"selected.test.js"}, nil); err != nil { t.Fatal(err) } @@ -522,7 +518,7 @@ func TestJestSeparatorPreservesOptionsAndReplacesSelection(t *testing.T) { if _, err := j.DiscoverTestFiles(t.Context(), discovery.TestFileSet{Pattern: "**/*.test.js"}); err != nil { t.Fatal(err) } - want = []string{"--runInBand", "--listTests", "--", "old.test.js"} + want = []string{"--runInBand", "--listTests", "--json", "--", "old.test.js"} if override[0] == "npx" { want = append([]string{"--", "jest"}, want...) } @@ -535,3 +531,50 @@ func TestJestSeparatorPreservesOptionsAndReplacesSelection(t *testing.T) { } } } + +func jestListOutput(paths ...string) []byte { + output, err := json.Marshal(paths) + if err != nil { + panic(err) + } + return output +} + +func TestJestDiscoveryWithNoisyOutput(t *testing.T) { + root := t.TempDir() + root, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + for _, name := range []string{"one.test.js", "two.test.js"} { + if err := os.WriteFile(name, []byte("test"), 0600); err != nil { + t.Fatal(err) + } + } + list := string(jestListOutput(filepath.Join(root, "one.test.js"), filepath.Join(root, "two.test.js"))) + for _, tc := range []struct { + name, output, wantError string + want []string + }{ + {name: "unterminated startup and shutdown logs", output: "startupstartup" + list + "shutdown", want: []string{"one.test.js", "two.test.js"}}, + {name: "brackets and unrelated arrays", output: "[debug] [123] [\"logging\"]" + list + "[exit]", want: []string{"one.test.js", "two.test.js"}}, + {name: "empty suite", output: "startup[]shutdown", want: []string{}}, + {name: "missing result", output: "startupshutdown", wantError: "missing Jest JSON test list"}, + {name: "truncated result", output: `startup["/incomplete`, wantError: "missing Jest JSON test list"}, + {name: "ambiguous result", output: "[]" + list, wantError: "ambiguous Jest JSON test list"}, + {name: "missing file", output: string(jestListOutput(filepath.Join(root, "missing.test.js"))), wantError: "invalid Jest test file"}, + } { + t.Run(tc.name, func(t *testing.T) { + jest := &Jest{executor: &jestCommandExecutor{output: []byte(tc.output)}} + files, err := jest.DiscoverTestFiles(t.Context(), discovery.TestFileSet{Pattern: "**/*.test.js"}) + if tc.wantError != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantError) { + t.Fatalf("got files %v, error %v; want %s", files, err, tc.wantError) + } + } else if err != nil || !slices.Equal(files, tc.want) { + t.Fatalf("got %v, %v; want %v", files, err, tc.want) + } + }) + } +} diff --git a/internal/testdrive/intake/settings.go b/internal/testdrive/intake/settings.go index 40327b48..dc5a0a36 100644 --- a/internal/testdrive/intake/settings.go +++ b/internal/testdrive/intake/settings.go @@ -67,12 +67,15 @@ func handleSettings(w http.ResponseWriter, request *http.Request) { response.Data.Attributes = api.SettingsResponseData{ CodeCoverage: true, CoverageReportUploadEnabled: true, - TestsSkipping: true, - ItrEnabled: true, - ImpactedTestsEnabled: true, - FlakyTestRetriesEnabled: true, - DIEnabled: true, - KnownTestsEnabled: true, + // Testdrive runs the whole suite and disables git upload. Some tracers + // wait for that upload before fetching skippable tests, so advertising + // skipping here can prevent Jest from ever starting. + TestsSkipping: false, + ItrEnabled: true, + ImpactedTestsEnabled: true, + FlakyTestRetriesEnabled: true, + DIEnabled: true, + KnownTestsEnabled: true, EarlyFlakeDetection: api.EarlyFlakeDetectionSettings{ Enabled: true, SlowTestRetries: api.SlowTestRetries{FiveS: 1, TenS: 1, ThirtyS: 1, FiveM: 1}, diff --git a/internal/testdrive/intake/settings_test.go b/internal/testdrive/intake/settings_test.go index 704bfefa..56a99d0f 100644 --- a/internal/testdrive/intake/settings_test.go +++ b/internal/testdrive/intake/settings_test.go @@ -44,7 +44,7 @@ func TestSettingsEnablesTestOptimizationCoverage(t *testing.T) { require.Equal(t, constants.SettingsResponseType, settings.Data.Type) require.True(t, settings.Data.Attributes.ItrEnabled) require.True(t, settings.Data.Attributes.CodeCoverage) - require.True(t, settings.Data.Attributes.TestsSkipping) + require.False(t, settings.Data.Attributes.TestsSkipping) require.False(t, settings.Data.Attributes.RequireGit) require.True(t, settings.Data.Attributes.CoverageReportUploadEnabled) require.True(t, settings.Data.Attributes.ImpactedTestsEnabled)