From faf2310f1e913c25669678bf7e3d4fc193c1b493 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 16:59:12 +0200 Subject: [PATCH 01/10] fix: enforce exact Vitest file assignments --- docs/examples/issue-168/README.md | 165 ++++++++++++++++++ docs/examples/issue-168/reproduce.mjs | 132 ++++++++++++++ internal/compatibility/vitest_test.go | 57 ++++++ .../framework/scripts/vitest_exact_files.mjs | 37 ++++ internal/framework/vitest.go | 53 +++++- internal/framework/vitest_test.go | 56 ++++++ 6 files changed, 499 insertions(+), 1 deletion(-) create mode 100644 docs/examples/issue-168/README.md create mode 100644 docs/examples/issue-168/reproduce.mjs create mode 100644 internal/framework/scripts/vitest_exact_files.mjs diff --git a/docs/examples/issue-168/README.md b/docs/examples/issue-168/README.md new file mode 100644 index 00000000..cc9e84f0 --- /dev/null +++ b/docs/examples/issue-168/README.md @@ -0,0 +1,165 @@ +# Issue #168 reproducer and impact assessment + +[Issue #168](https://github.com/DataDog/ddtest/issues/168) is reproducible on +ddtest source commit `de191d143` with Node 24.14.0, Vitest 5.0.1, and dd-trace +5.111.0 on macOS arm64. The investigation was performed on 2026-10-07. +This validates the current source; the issue's released v1.11.0 binary and hosted +date-fns run were not independently rerun. + +## Run the reproducer + +Prerequisites: the Go version in `go.mod`, Node 24.14.0 on PATH, npm, Git, and +access to the npm registry. No Datadog credentials are needed. + +From the repository root: + +```sh +go build -o /tmp/ddtest-issue-168 main.go +node docs/examples/issue-168/reproduce.mjs /tmp/ddtest-issue-168 +``` + +The script creates an isolated temporary Git repository, installs pinned direct +dependencies, and generates two one-test files: + +```text +src/endOfYear/test.ts +src/eachWeekendOfYear/test.ts +``` + +Its Vitest configuration is: + +```js +export default { test: { dir: 'src', include: ['**/test.ts'] } } +``` + +The script disables Datadog CI Visibility and tracing for these subprocesses. +Every test appends its file name, process ID, and ddtest worker session to a +JSONL file **inside the test body**. Counts therefore reflect actual test +execution, independently of ddtest's summary. Each scenario starts with a fresh +plan and separate local resources. + +The script now exits zero when all eight scenarios validate the fix: ddtest +runs each assigned file exactly once, honors selection/exclusion, and passes +the create-once test. Direct Vitest still demonstrates substring filtering. +To reproduce the original behavior with an unfixed binary, append `--expect-bug`: + +```sh +node docs/examples/issue-168/reproduce.mjs /absolute/path/to/unfixed-ddtest --expect-bug +``` + +It prints the temporary directory containing the generated +fixture, npm lockfile, `versions.txt`, `results.json`, per-scenario logs, +execution records, and saved worker assignments. Delete that directory when +finished; it is retained for inspection. + +## Observed results before the fix + +Each file contains exactly one test. Expected counts below follow the user's +selection; actual counts come from the execution records. + +| Scenario | Expected executions | Actual executions | Exit code | +| --- | ---: | ---: | ---: | +| Ordinary Vitest, all files | 2 | 2 | 0 | +| Vitest, relative `endOfYear` file filter | 1 | 2 | 0 | +| Vitest, canonical absolute `endOfYear` file filter | 1 | 2 | 0 | +| ddtest, two workers | 2 | 3 | 0 | +| ddtest, explicit selection of `endOfYear` only | 1 | 2 | 0 | +| ddtest, exclude `eachWeekendOfYear` | 1 | 2 | 0 | +| Ordinary Vitest, create-once resources | 2 | 2 | 0 | +| ddtest, two workers and create-once resources | 2 | 3 | 1 | + +The parallel run saved this correct, disjoint plan: + +```text +runner-0: src/eachWeekendOfYear/test.ts +runner-1: src/endOfYear/test.ts +``` + +Execution records show `eachWeekendOfYear` running in both worker sessions. +Vitest reports one passed file in runner 0 and two passed files in runner 1. +ddtest still reports `Test files run: 2` and `Result: passed`. +The explicit-selection and exclusion scenarios each plan only `endOfYear`, run +both files, and report `Test files run: 1`. + +## Cause + +`Vitest.RunTests` in `internal/framework/vitest.go` uses +`withFrameworkFiles` in `internal/framework/command_args.go` to append the +assigned file paths as positional arguments. Vitest documents that these are +[substring filename filters](https://vitest.dev/guide/filtering#filtering-by-file-name). + +In the installed Vitest 5.0.1 implementation, `filterFiles` also compares paths +relative to the configured test directory and lowercases them: + +```text +filter: endofyear/test.ts +candidate: eachweekendofyear/test.ts +``` + +The candidate contains the filter. Selecting the shorter file consequently +selects the longer file as well. Canonical absolute paths still reproduce this +because the implementation retains the relative substring fallback. + +The planner assigns the files correctly; execution expands its assignments. +The report counts the selected batch lengths in +`internal/runner/parallel_executor.go` and `sequential_executor.go`, rather than +measuring the files actually executed by Vitest. + +## Impact and simple failure example + +This is an execution correctness problem. In the two-file fixture it adds 50% +more test executions and understates the actual work. Across more workers, +one file can execute in every worker whose filter matches it. Multiple matching +filters in one Vitest invocation do not themselves imply multiple executions +there; the observed duplication is between worker invocations. + +The same expansion can violate an explicit selection or ddtest exclusion even +with just one worker. Pure tests can remain green while wasting CPU and API +calls. Tests with shared resources can produce duplicate side effects, contention, +or failures; intended shard isolation no longer holds. + +A simple example is a test in `eachWeekendOfYear/test.ts` that creates a database +row with a unique key. Both workers execute it, so the second insertion fails. +The reproducer models that without a database, using an exclusive file creation: + +```js +writeFileSync(resourcePath, 'created', { flag: 'wx' }) +``` + +Each test has its own resource path, so ordinary Vitest passes. Under ddtest the +duplicated test attempts to create the same resource twice, causing `EEXIST` +and exit code 1. This failure is deterministic regardless of which worker wins +the first creation; it needs no timing-based race or flaky application code. + +An unrestricted single-worker run of both files executes each once in this +fixture. The issue requires overlapping filters that cross an assignment or +selection boundary. No omissions were observed in these scenarios. +Authenticated tracer/TIA enforcement was not tested, so its effect on the +extra test executions remains unverified. + +## Fix + +The Vitest adapter now installs a Node preload that restricts Vitest's file +filter to exact membership in the worker's assigned files. Both assigned paths +and candidates are resolved to canonical absolute paths without lowercasing. +The original CLI still owns the real run, including its configuration, command +wrappers, reporters, coverage, tracer hooks, and failure exit codes. + +Vitest does not export its project class on every supported version. The preload +obtains the shared project prototype from an empty context rooted in the +adapter's temporary directory, with config loading disabled, then closes that +context before the real CLI starts. It does not load the user's configuration +or workspace for that preliminary context. The exact filter applies to every +project in the subsequent real run. An unsupported API fails explicitly rather +than falling back to fuzzy selection. Empty batches do not invoke Vitest. + +The assigned paths are passed through a temporary JSON file to avoid putting a +large allowlist into the environment. The adapter and list are removed after +the worker finishes, including on test failure. + +The integration regression in `internal/compatibility/vitest_test.go` runs in +the existing Vitest CI matrix. It checks overlapping names under `test.dir`, +relative and absolute assignments, direct and package-manager commands, +end-of-options separators, configured setup files, worker environment, and +failure exit status. The executable repro additionally validates parallel +execution, explicit selection, exclusion, and create-once side effects. diff --git a/docs/examples/issue-168/reproduce.mjs b/docs/examples/issue-168/reproduce.mjs new file mode 100644 index 00000000..2ced20ce --- /dev/null +++ b/docs/examples/issue-168/reproduce.mjs @@ -0,0 +1,132 @@ +// Run with Node 24.14.0 and an absolute path to the ddtest binary: +// node docs/examples/issue-168/reproduce.mjs /absolute/path/to/ddtest +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' + +const binary = resolve(process.argv[2] || './ddtest') +const expectBug = process.argv.includes('--expect-bug') +assert.ok(existsSync(binary), `Build ddtest first; binary not found: ${binary}`) +// Use the canonical path (macOS aliases /var to /private/var). +const fixture = realpathSync(mkdtempSync(join(tmpdir(), 'ddtest-issue-168-'))) +console.log(`Fixture, logs, and execution records: ${fixture}`) + +function write(path, content) { + const target = join(fixture, path) + mkdirSync(dirname(target), { recursive: true }) + writeFileSync(target, content) +} + +// Disable backend/tracer use and isolate the experiment from local ddtest settings. +const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => + !key.startsWith('DD_TEST_OPTIMIZATION_') && !key.startsWith('DD_TESTOPTIMIZATION_'))) +Object.assign(env, { + NODE_OPTIONS: '', DD_API_KEY: '', DD_APP_KEY: '', DD_CIVISIBILITY_ENABLED: 'false', + DD_TRACE_ENABLED: 'false', DD_SERVICE: 'issue-168-repro', + DD_TRACE_PACKAGE: join(fixture, 'node_modules/dd-trace/ci/init'), + DD_TRACE_ESM_IMPORT: join(fixture, 'node_modules/dd-trace/register.js'), +}) + +function run(command, args, extraEnv = {}) { + const result = spawnSync(command, args, { + cwd: fixture, env: { ...env, ...extraEnv }, encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, + }) + if (result.error) throw result.error + assert.notEqual(result.status, null, `Command terminated by ${result.signal}`) + return result +} + +function setup(command, args) { + const result = run(command, args) + assert.equal(result.status, 0, `${command} failed:\n${result.stdout}\n${result.stderr}`) +} + +write('package.json', JSON.stringify({ + private: true, type: 'module', devDependencies: { vitest: '5.0.1', 'dd-trace': '5.111.0' }, +}, null, 2)) +write('vitest.config.mjs', 'export default { test: { dir: "src", include: ["**/test.ts"] } };\n') +for (const name of ['endOfYear', 'eachWeekendOfYear']) { + write(`src/${name}/test.ts`, ` +import { test, expect } from 'vitest'; +import { appendFileSync, writeFileSync } from 'node:fs'; +test('${name}', () => { + appendFileSync(process.env.REPRO_EVENTS, JSON.stringify({ + file: 'src/${name}/test.ts', pid: process.pid, + session: process.env.DD_TEST_SESSION_NAME || 'direct', + }) + '\\n'); + // Model an external resource with a unique key: a second creation must fail. + if (process.env.REPRO_ONCE === '1') + writeFileSync(process.env.REPRO_ONCE_DIR + '/${name}', 'created', { flag: 'wx' }); + expect(1).toBe(1); +}); +`) +} +setup('git', ['init', '-q']) +setup('git', ['-c', 'user.name=Reproducer', '-c', 'user.email=reproducer@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '--allow-empty', '-qm', 'Initialize reproducer']) +console.log('Installing pinned Vitest and tracer dependencies...') +setup('npm', ['install', '--no-audit', '--no-fund', '--cache', join(fixture, 'npm-cache')]) +write('versions.txt', `Node ${process.version}\nddtest ${run(binary, ['--version']).stdout.trim()}\nVitest 5.0.1\ndd-trace 5.111.0\n`) + +const vitest = ['node_modules/vitest/vitest.mjs', 'run'] +const common = ['run', '--platform', 'javascript', '--framework', 'vitest', + '--command', 'node node_modules/vitest/vitest.mjs run'] +const selected = 'src/endOfYear/test.ts' +const other = 'src/eachWeekendOfYear/test.ts' +const summaries = [] + +function scenario(name, command, args, expectedCounts, shouldPass, once = false) { + rmSync(join(fixture, '.testoptimization'), { recursive: true, force: true }) + const eventsPath = join(fixture, `${name}.jsonl`) + writeFileSync(eventsPath, '') + const onceDir = join(fixture, `${name}-resources`) + mkdirSync(onceDir) + const result = run(command, args, { + REPRO_EVENTS: eventsPath, REPRO_ONCE: once ? '1' : '0', REPRO_ONCE_DIR: onceDir, + }) + write(`${name}.log`, result.stdout + result.stderr) + const events = readFileSync(eventsPath, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse) + const counts = Object.fromEntries([selected, other].map(file => + [file, events.filter(event => event.file === file).length])) + let assignments + const splitDir = join(fixture, '.testoptimization/runner/tests-split') + if (existsSync(splitDir)) { + assignments = Object.fromEntries(readdirSync(splitDir).map(file => + [file, readFileSync(join(splitDir, file), 'utf8').trim().split('\n').filter(Boolean)])) + write(`${name}-assignments.json`, JSON.stringify(assignments, null, 2)) + } + const summary = { name, exitCode: result.status, counts, events, assignments } + summaries.push(summary) + write('results.json', JSON.stringify(summaries, null, 2)) + console.log(`${name}: exit=${result.status}, endOfYear=${counts[selected]}, eachWeekendOfYear=${counts[other]}`) + assert.deepEqual(counts, { [selected]: expectedCounts[0], [other]: expectedCounts[1] }, + `Unexpected executions; inspect ${name}.log.`) + assert.equal(result.status === 0, shouldPass, `Unexpected exit status; inspect ${name}.log`) + return result +} + +scenario('baseline', process.execPath, vitest, [1, 1], true) +scenario('relative-filter', process.execPath, [...vitest, selected], [1, 1], true) +scenario('absolute-filter', process.execPath, [...vitest, join(fixture, selected)], [1, 1], true) +const parallel = scenario('ddtest-parallel', binary, + [...common, '--min-parallelism', '2', '--max-parallelism', '2'], expectBug ? [1, 2] : [1, 1], true) +assert.match(parallel.stdout + parallel.stderr, /Test files run: 2/) +const assignmentFiles = Object.values(summaries.at(-1).assignments) +assert.equal(assignmentFiles.length, 2) +assert.ok(assignmentFiles.every(files => files.length === 1)) +assert.deepEqual(assignmentFiles.flat().sort(), [selected, other].sort()) +scenario('ddtest-explicit-selection', binary, + [...common, '--min-parallelism', '1', '--max-parallelism', '1', '--', selected], expectBug ? [1, 1] : [1, 0], true) +scenario('ddtest-exclude', binary, + [...common, '--min-parallelism', '1', '--max-parallelism', '1', + '--tests-exclude-pattern', other], expectBug ? [1, 1] : [1, 0], true) +scenario('baseline-create-once', process.execPath, vitest, [1, 1], true, true) +const createOnce = scenario('ddtest-create-once', binary, + [...common, '--min-parallelism', '2', '--max-parallelism', '2'], expectBug ? [1, 2] : [1, 1], !expectBug, true) +if (expectBug) assert.match(createOnce.stdout + createOnce.stderr, /EEXIST/) +console.log(expectBug + ? 'Confirmed: disjoint assignments expand into duplicate execution; create-once test fails only under ddtest.' + : 'Verified: exact assignments, explicit selection, exclusion, and create-once tests all pass.') +console.log(`Evidence retained in ${fixture}; remove that directory when finished.`) diff --git a/internal/compatibility/vitest_test.go b/internal/compatibility/vitest_test.go index d2868843..662efee8 100644 --- a/internal/compatibility/vitest_test.go +++ b/internal/compatibility/vitest_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "time" @@ -58,4 +59,60 @@ test('must not run', () => { if err := vitest.RunTests(ctx, []string{"checks/selected.check.js"}, map[string]string{"DDTEST_VITEST_WORKER": "selected"}); err != nil { t.Fatalf("selected-file run failed: %v", err) } + + t.Run("exact file membership with overlapping names", func(t *testing.T) { + writeFixture(t, root, "vitest.overlap.mjs", `export default { + test: { dir: 'src', include: ['**/test.js'], setupFiles: ['./setup.js'] }, +} +`) + for _, name := range []string{"endOfYear", "eachWeekendOfYear", "otherEndOfYear"} { + writeFixture(t, root, "src/"+name+"/test.js", `import { expect, test } from 'vitest' +import { appendFileSync } from 'node:fs' +test('runs only in its assigned batch', () => { + expect(globalThis.ddtestVitestSetup).toBe(true) + expect(process.env.DDTEST_VITEST_WORKER).toBe('`+name+`') + appendFileSync(process.env.DDTEST_VITEST_EVENTS, '`+name+`\n') +}) +`) + } + for _, tc := range []struct { + name, command, selected string + }{ + {"direct", shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.overlap.mjs"), "src/endOfYear/test.js"}, + {"node with separator", shellCommand("node", filepath.Join(root, "node_modules/vitest/vitest.mjs"), "run", "--config", "vitest.overlap.mjs", "--", "old.test.js"), filepath.Join(root, "src/endOfYear/test.js")}, + {"package manager", shellCommand("npx", "--no-install", "vitest", "run", "--config", "vitest.overlap.mjs"), "src/eachWeekendOfYear/test.js"}, + } { + t.Run(tc.name, func(t *testing.T) { + events := filepath.Join(root, "executed.txt") + if err := os.WriteFile(events, nil, 0600); err != nil { + t.Fatal(err) + } + configureFramework(tc.command, "") + name := filepath.Base(filepath.Dir(tc.selected)) + if err := framework.NewVitest().RunTests(ctx, []string{tc.selected}, map[string]string{ + "DDTEST_VITEST_WORKER": name, "DDTEST_VITEST_EVENTS": events, + }); err != nil { + t.Fatalf("exact-file run failed: %v", err) + } + contents, err := os.ReadFile(events) + if err != nil { + t.Fatal(err) + } + if string(contents) != name+"\n" { + t.Fatalf("executed %q, want exactly %q", contents, name+"\n") + } + }) + } + + // An empty batch must not turn into an unfiltered full-suite run. + configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.overlap.mjs"), "") + if err := framework.NewVitest().RunTests(ctx, nil, nil); err != nil { + t.Fatal(err) + } + // Preserve the framework's nonzero exit status on an assigned test failure. + err := framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, map[string]string{"DDTEST_VITEST_WORKER": "wrong"}) + if err == nil || !strings.Contains(err.Error(), "exit status 1") { + t.Fatalf("expected assigned-test failure, got %v", err) + } + }) } diff --git a/internal/framework/scripts/vitest_exact_files.mjs b/internal/framework/scripts/vitest_exact_files.mjs new file mode 100644 index 00000000..703f4b5f --- /dev/null +++ b/internal/framework/scripts/vitest_exact_files.mjs @@ -0,0 +1,37 @@ +import { readFileSync, realpathSync } from 'node:fs' +import { createRequire } from 'node:module' +import { basename, dirname, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +const entrypoint = process.argv[1] || '' +const filesPath = process.env.DDTEST_VITEST_SELECTED_FILES + +// Package managers inherit NODE_OPTIONS. Wait for the actual Vitest process, +// then clear the request so its test workers do not install the adapter again. +if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypoint))) { + delete process.env.DDTEST_VITEST_SELECTED_FILES + const canonicalPath = file => realpathSync(resolve(file)) + const selected = new Set(JSON.parse(readFileSync(filesPath, 'utf8')).map(canonicalPath)) + const require = createRequire(pathToFileURL(realpathSync(entrypoint))) + const { createVitest } = await import(pathToFileURL(require.resolve('vitest/node')).href) + + // Vitest does not export the project class on every supported version. An + // empty, config-free context gives us its prototype without loading the + // user's configuration twice. The original CLI still owns the real run, + // including reporters, coverage, tracer hooks, exit codes, and teardown. + const context = await createVitest('test', { + root: dirname(filesPath), config: false, watch: false, include: [], + }) + try { + const project = context.projects[0] + if (!project || typeof project.filterFiles !== 'function') { + throw new Error('ddtest cannot enforce exact file selection with this Vitest version') + } + const prototype = Object.getPrototypeOf(project) + prototype.filterFiles = function (files) { + return files.filter(file => selected.has(canonicalPath(file))) + } + } finally { + await context.close() + } +} diff --git a/internal/framework/vitest.go b/internal/framework/vitest.go index a0873f2e..74bbfb67 100644 --- a/internal/framework/vitest.go +++ b/internal/framework/vitest.go @@ -8,9 +8,11 @@ import ( "fmt" "log/slog" "maps" + "net/url" "os" "path/filepath" "slices" + "strconv" "strings" "github.com/DataDog/ddtest/internal/discovery" @@ -24,11 +26,15 @@ const ( binVitestPath = "node_modules/.bin/vitest" ddTraceRegisterPath = "dd-trace/register.js" vitestV1DiscoveryMarker = "__DDTEST_VITEST_FILES__" + vitestSelectedFilesEnv = "DDTEST_VITEST_SELECTED_FILES" ) //go:embed scripts/vitest_v1_discovery.mjs var vitestV1DiscoveryScript string +//go:embed scripts/vitest_exact_files.mjs +var vitestExactFilesScript string + var vitestTestFileExtensions = []string{"js", "jsx", "ts", "tsx", "mjs", "mts", "cjs", "cts"} type vitestExecutor interface { @@ -191,6 +197,9 @@ func (v *Vitest) discoverVitestV1TestFiles(ctx context.Context, command string, } func (v *Vitest) RunTests(ctx context.Context, testFiles []string, envMap map[string]string) error { + if len(testFiles) == 0 { + return nil + } command, baseArgs := v.Command() args := vitestArgsForSubcommand(baseArgs, "run") args = withFrameworkFiles(command, args, "vitest", testFiles) @@ -200,7 +209,49 @@ func (v *Vitest) RunTests(ctx context.Context, testFiles []string, envMap map[st mergedEnv := make(map[string]string) maps.Copy(mergedEnv, v.platformEnv) maps.Copy(mergedEnv, envMap) - return v.executor.Run(ctx, command, args, mergedEnv) + adapterDir, adapterEnv, err := prepareVitestExactFiles(mergedEnv, testFiles) + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(adapterDir) }() + return v.executor.Run(ctx, command, args, adapterEnv) +} + +func prepareVitestExactFiles(baseEnv map[string]string, testFiles []string) (string, map[string]string, error) { + selectedFiles := make([]string, len(testFiles)) + for i, file := range testFiles { + absolute, err := filepath.Abs(file) + if err != nil { + return "", nil, fmt.Errorf("failed to resolve Vitest test file: %w", err) + } + selectedFiles[i] = absolute + } + encodedFiles, err := json.Marshal(selectedFiles) + if err != nil { + return "", nil, fmt.Errorf("failed to encode Vitest test files: %w", err) + } + adapterDir, err := os.MkdirTemp("", "ddtest-vitest-run-*") + if err != nil { + return "", nil, fmt.Errorf("failed to create Vitest execution adapter: %w", err) + } + adapterPath := filepath.Join(adapterDir, "exact-files.mjs") + filesPath := filepath.Join(adapterDir, "files.json") + for path, contents := range map[string][]byte{adapterPath: []byte(vitestExactFilesScript), filesPath: encodedFiles} { + if err := os.WriteFile(path, contents, 0600); err != nil { + _ = os.RemoveAll(adapterDir) + return "", nil, fmt.Errorf("failed to write Vitest execution adapter: %w", err) + } + } + adapterEnv := make(map[string]string, len(baseEnv)+2) + maps.Copy(adapterEnv, baseEnv) + nodeOptions, ok := adapterEnv[nodeOptionsEnvVar] + if !ok { + nodeOptions = os.Getenv(nodeOptionsEnvVar) + } + importURL := &url.URL{Scheme: "file", Path: "/" + strings.TrimPrefix(filepath.ToSlash(adapterPath), "/")} + adapterEnv[nodeOptionsEnvVar] = strings.TrimSpace(nodeOptions + " --import " + strconv.Quote(importURL.String())) + adapterEnv[vitestSelectedFilesEnv] = filesPath + return adapterDir, adapterEnv, nil } func (v *Vitest) discoveryEnv() map[string]string { diff --git a/internal/framework/vitest_test.go b/internal/framework/vitest_test.go index 5f7d438b..5a1a9ac1 100644 --- a/internal/framework/vitest_test.go +++ b/internal/framework/vitest_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "path/filepath" "slices" @@ -15,6 +16,7 @@ import ( ) type vitestCommandExecutor struct { + onRun func(string, []string, map[string]string) error output []byte stdout []byte stderr []byte @@ -92,6 +94,9 @@ func (m *vitestCommandExecutor) Run(_ context.Context, name string, args []strin m.capturedName = name m.capturedArgs = slices.Clone(args) m.capturedEnvMap = envMap + if m.onRun != nil { + return m.onRun(name, args, envMap) + } return m.err } @@ -475,6 +480,57 @@ func TestVitest_RunTests_UsesNpxFallback(t *testing.T) { } } +func TestVitest_RunTests_ExactSelectionEnvironmentAndCleanup(t *testing.T) { + for _, runErr := range []error{nil, errors.New("test process failed")} { + t.Run(fmt.Sprintf("run error %v", runErr), func(t *testing.T) { + var selectionPath string + workerEnv := map[string]string{"NODE_OPTIONS": "--max-old-space-size=2048"} + executor := &vitestCommandExecutor{onRun: func(_ string, _ []string, env map[string]string) error { + selectionPath = env[vitestSelectedFilesEnv] + contents, err := os.ReadFile(selectionPath) + if err != nil { + t.Fatal(err) + } + var files []string + if err := json.Unmarshal(contents, &files); err != nil { + t.Fatal(err) + } + absolute, err := filepath.Abs("src/endOfYear/test.ts") + if err != nil || !slices.Equal(files, []string{absolute}) { + t.Fatalf("selected files = %v, error = %v", files, err) + } + if !strings.HasPrefix(env["NODE_OPTIONS"], workerEnv["NODE_OPTIONS"]+" --import ") { + t.Fatalf("lost worker NODE_OPTIONS: %q", env["NODE_OPTIONS"]) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(selectionPath), "exact-files.mjs")); err != nil { + t.Fatal(err) + } + return runErr + }} + vitest := &Vitest{executor: executor, platformEnv: map[string]string{"NODE_OPTIONS": "platform-options"}} + if err := vitest.RunTests(t.Context(), []string{"src/endOfYear/test.ts"}, workerEnv); !errors.Is(err, runErr) { + t.Fatalf("got %v, want %v", err, runErr) + } + if _, err := os.Stat(filepath.Dir(selectionPath)); !os.IsNotExist(err) { + t.Fatalf("adapter directory was not cleaned up: %v", err) + } + if len(workerEnv) != 1 || workerEnv["NODE_OPTIONS"] != "--max-old-space-size=2048" { + t.Fatalf("mutated worker environment: %v", workerEnv) + } + }) + } +} + +func TestVitest_RunTests_EmptyBatch(t *testing.T) { + executor := &vitestCommandExecutor{onRun: func(string, []string, map[string]string) error { + t.Fatal("empty batch must not invoke Vitest") + return nil + }} + if err := (&Vitest{executor: executor}).RunTests(t.Context(), nil, nil); err != nil { + t.Fatal(err) + } +} + func TestVitestArgsForSubcommand(t *testing.T) { tests := []struct { name string From 0e076b8693430dda5876bd4bb050cc309392a769 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 17:06:35 +0200 Subject: [PATCH 02/10] Remove issue examples from Vitest fix PR --- docs/examples/issue-168/README.md | 165 -------------------------- docs/examples/issue-168/reproduce.mjs | 132 --------------------- 2 files changed, 297 deletions(-) delete mode 100644 docs/examples/issue-168/README.md delete mode 100644 docs/examples/issue-168/reproduce.mjs diff --git a/docs/examples/issue-168/README.md b/docs/examples/issue-168/README.md deleted file mode 100644 index cc9e84f0..00000000 --- a/docs/examples/issue-168/README.md +++ /dev/null @@ -1,165 +0,0 @@ -# Issue #168 reproducer and impact assessment - -[Issue #168](https://github.com/DataDog/ddtest/issues/168) is reproducible on -ddtest source commit `de191d143` with Node 24.14.0, Vitest 5.0.1, and dd-trace -5.111.0 on macOS arm64. The investigation was performed on 2026-10-07. -This validates the current source; the issue's released v1.11.0 binary and hosted -date-fns run were not independently rerun. - -## Run the reproducer - -Prerequisites: the Go version in `go.mod`, Node 24.14.0 on PATH, npm, Git, and -access to the npm registry. No Datadog credentials are needed. - -From the repository root: - -```sh -go build -o /tmp/ddtest-issue-168 main.go -node docs/examples/issue-168/reproduce.mjs /tmp/ddtest-issue-168 -``` - -The script creates an isolated temporary Git repository, installs pinned direct -dependencies, and generates two one-test files: - -```text -src/endOfYear/test.ts -src/eachWeekendOfYear/test.ts -``` - -Its Vitest configuration is: - -```js -export default { test: { dir: 'src', include: ['**/test.ts'] } } -``` - -The script disables Datadog CI Visibility and tracing for these subprocesses. -Every test appends its file name, process ID, and ddtest worker session to a -JSONL file **inside the test body**. Counts therefore reflect actual test -execution, independently of ddtest's summary. Each scenario starts with a fresh -plan and separate local resources. - -The script now exits zero when all eight scenarios validate the fix: ddtest -runs each assigned file exactly once, honors selection/exclusion, and passes -the create-once test. Direct Vitest still demonstrates substring filtering. -To reproduce the original behavior with an unfixed binary, append `--expect-bug`: - -```sh -node docs/examples/issue-168/reproduce.mjs /absolute/path/to/unfixed-ddtest --expect-bug -``` - -It prints the temporary directory containing the generated -fixture, npm lockfile, `versions.txt`, `results.json`, per-scenario logs, -execution records, and saved worker assignments. Delete that directory when -finished; it is retained for inspection. - -## Observed results before the fix - -Each file contains exactly one test. Expected counts below follow the user's -selection; actual counts come from the execution records. - -| Scenario | Expected executions | Actual executions | Exit code | -| --- | ---: | ---: | ---: | -| Ordinary Vitest, all files | 2 | 2 | 0 | -| Vitest, relative `endOfYear` file filter | 1 | 2 | 0 | -| Vitest, canonical absolute `endOfYear` file filter | 1 | 2 | 0 | -| ddtest, two workers | 2 | 3 | 0 | -| ddtest, explicit selection of `endOfYear` only | 1 | 2 | 0 | -| ddtest, exclude `eachWeekendOfYear` | 1 | 2 | 0 | -| Ordinary Vitest, create-once resources | 2 | 2 | 0 | -| ddtest, two workers and create-once resources | 2 | 3 | 1 | - -The parallel run saved this correct, disjoint plan: - -```text -runner-0: src/eachWeekendOfYear/test.ts -runner-1: src/endOfYear/test.ts -``` - -Execution records show `eachWeekendOfYear` running in both worker sessions. -Vitest reports one passed file in runner 0 and two passed files in runner 1. -ddtest still reports `Test files run: 2` and `Result: passed`. -The explicit-selection and exclusion scenarios each plan only `endOfYear`, run -both files, and report `Test files run: 1`. - -## Cause - -`Vitest.RunTests` in `internal/framework/vitest.go` uses -`withFrameworkFiles` in `internal/framework/command_args.go` to append the -assigned file paths as positional arguments. Vitest documents that these are -[substring filename filters](https://vitest.dev/guide/filtering#filtering-by-file-name). - -In the installed Vitest 5.0.1 implementation, `filterFiles` also compares paths -relative to the configured test directory and lowercases them: - -```text -filter: endofyear/test.ts -candidate: eachweekendofyear/test.ts -``` - -The candidate contains the filter. Selecting the shorter file consequently -selects the longer file as well. Canonical absolute paths still reproduce this -because the implementation retains the relative substring fallback. - -The planner assigns the files correctly; execution expands its assignments. -The report counts the selected batch lengths in -`internal/runner/parallel_executor.go` and `sequential_executor.go`, rather than -measuring the files actually executed by Vitest. - -## Impact and simple failure example - -This is an execution correctness problem. In the two-file fixture it adds 50% -more test executions and understates the actual work. Across more workers, -one file can execute in every worker whose filter matches it. Multiple matching -filters in one Vitest invocation do not themselves imply multiple executions -there; the observed duplication is between worker invocations. - -The same expansion can violate an explicit selection or ddtest exclusion even -with just one worker. Pure tests can remain green while wasting CPU and API -calls. Tests with shared resources can produce duplicate side effects, contention, -or failures; intended shard isolation no longer holds. - -A simple example is a test in `eachWeekendOfYear/test.ts` that creates a database -row with a unique key. Both workers execute it, so the second insertion fails. -The reproducer models that without a database, using an exclusive file creation: - -```js -writeFileSync(resourcePath, 'created', { flag: 'wx' }) -``` - -Each test has its own resource path, so ordinary Vitest passes. Under ddtest the -duplicated test attempts to create the same resource twice, causing `EEXIST` -and exit code 1. This failure is deterministic regardless of which worker wins -the first creation; it needs no timing-based race or flaky application code. - -An unrestricted single-worker run of both files executes each once in this -fixture. The issue requires overlapping filters that cross an assignment or -selection boundary. No omissions were observed in these scenarios. -Authenticated tracer/TIA enforcement was not tested, so its effect on the -extra test executions remains unverified. - -## Fix - -The Vitest adapter now installs a Node preload that restricts Vitest's file -filter to exact membership in the worker's assigned files. Both assigned paths -and candidates are resolved to canonical absolute paths without lowercasing. -The original CLI still owns the real run, including its configuration, command -wrappers, reporters, coverage, tracer hooks, and failure exit codes. - -Vitest does not export its project class on every supported version. The preload -obtains the shared project prototype from an empty context rooted in the -adapter's temporary directory, with config loading disabled, then closes that -context before the real CLI starts. It does not load the user's configuration -or workspace for that preliminary context. The exact filter applies to every -project in the subsequent real run. An unsupported API fails explicitly rather -than falling back to fuzzy selection. Empty batches do not invoke Vitest. - -The assigned paths are passed through a temporary JSON file to avoid putting a -large allowlist into the environment. The adapter and list are removed after -the worker finishes, including on test failure. - -The integration regression in `internal/compatibility/vitest_test.go` runs in -the existing Vitest CI matrix. It checks overlapping names under `test.dir`, -relative and absolute assignments, direct and package-manager commands, -end-of-options separators, configured setup files, worker environment, and -failure exit status. The executable repro additionally validates parallel -execution, explicit selection, exclusion, and create-once side effects. diff --git a/docs/examples/issue-168/reproduce.mjs b/docs/examples/issue-168/reproduce.mjs deleted file mode 100644 index 2ced20ce..00000000 --- a/docs/examples/issue-168/reproduce.mjs +++ /dev/null @@ -1,132 +0,0 @@ -// Run with Node 24.14.0 and an absolute path to the ddtest binary: -// node docs/examples/issue-168/reproduce.mjs /absolute/path/to/ddtest -import assert from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { dirname, join, resolve } from 'node:path' - -const binary = resolve(process.argv[2] || './ddtest') -const expectBug = process.argv.includes('--expect-bug') -assert.ok(existsSync(binary), `Build ddtest first; binary not found: ${binary}`) -// Use the canonical path (macOS aliases /var to /private/var). -const fixture = realpathSync(mkdtempSync(join(tmpdir(), 'ddtest-issue-168-'))) -console.log(`Fixture, logs, and execution records: ${fixture}`) - -function write(path, content) { - const target = join(fixture, path) - mkdirSync(dirname(target), { recursive: true }) - writeFileSync(target, content) -} - -// Disable backend/tracer use and isolate the experiment from local ddtest settings. -const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => - !key.startsWith('DD_TEST_OPTIMIZATION_') && !key.startsWith('DD_TESTOPTIMIZATION_'))) -Object.assign(env, { - NODE_OPTIONS: '', DD_API_KEY: '', DD_APP_KEY: '', DD_CIVISIBILITY_ENABLED: 'false', - DD_TRACE_ENABLED: 'false', DD_SERVICE: 'issue-168-repro', - DD_TRACE_PACKAGE: join(fixture, 'node_modules/dd-trace/ci/init'), - DD_TRACE_ESM_IMPORT: join(fixture, 'node_modules/dd-trace/register.js'), -}) - -function run(command, args, extraEnv = {}) { - const result = spawnSync(command, args, { - cwd: fixture, env: { ...env, ...extraEnv }, encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, - }) - if (result.error) throw result.error - assert.notEqual(result.status, null, `Command terminated by ${result.signal}`) - return result -} - -function setup(command, args) { - const result = run(command, args) - assert.equal(result.status, 0, `${command} failed:\n${result.stdout}\n${result.stderr}`) -} - -write('package.json', JSON.stringify({ - private: true, type: 'module', devDependencies: { vitest: '5.0.1', 'dd-trace': '5.111.0' }, -}, null, 2)) -write('vitest.config.mjs', 'export default { test: { dir: "src", include: ["**/test.ts"] } };\n') -for (const name of ['endOfYear', 'eachWeekendOfYear']) { - write(`src/${name}/test.ts`, ` -import { test, expect } from 'vitest'; -import { appendFileSync, writeFileSync } from 'node:fs'; -test('${name}', () => { - appendFileSync(process.env.REPRO_EVENTS, JSON.stringify({ - file: 'src/${name}/test.ts', pid: process.pid, - session: process.env.DD_TEST_SESSION_NAME || 'direct', - }) + '\\n'); - // Model an external resource with a unique key: a second creation must fail. - if (process.env.REPRO_ONCE === '1') - writeFileSync(process.env.REPRO_ONCE_DIR + '/${name}', 'created', { flag: 'wx' }); - expect(1).toBe(1); -}); -`) -} -setup('git', ['init', '-q']) -setup('git', ['-c', 'user.name=Reproducer', '-c', 'user.email=reproducer@example.invalid', - '-c', 'commit.gpgsign=false', 'commit', '--allow-empty', '-qm', 'Initialize reproducer']) -console.log('Installing pinned Vitest and tracer dependencies...') -setup('npm', ['install', '--no-audit', '--no-fund', '--cache', join(fixture, 'npm-cache')]) -write('versions.txt', `Node ${process.version}\nddtest ${run(binary, ['--version']).stdout.trim()}\nVitest 5.0.1\ndd-trace 5.111.0\n`) - -const vitest = ['node_modules/vitest/vitest.mjs', 'run'] -const common = ['run', '--platform', 'javascript', '--framework', 'vitest', - '--command', 'node node_modules/vitest/vitest.mjs run'] -const selected = 'src/endOfYear/test.ts' -const other = 'src/eachWeekendOfYear/test.ts' -const summaries = [] - -function scenario(name, command, args, expectedCounts, shouldPass, once = false) { - rmSync(join(fixture, '.testoptimization'), { recursive: true, force: true }) - const eventsPath = join(fixture, `${name}.jsonl`) - writeFileSync(eventsPath, '') - const onceDir = join(fixture, `${name}-resources`) - mkdirSync(onceDir) - const result = run(command, args, { - REPRO_EVENTS: eventsPath, REPRO_ONCE: once ? '1' : '0', REPRO_ONCE_DIR: onceDir, - }) - write(`${name}.log`, result.stdout + result.stderr) - const events = readFileSync(eventsPath, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse) - const counts = Object.fromEntries([selected, other].map(file => - [file, events.filter(event => event.file === file).length])) - let assignments - const splitDir = join(fixture, '.testoptimization/runner/tests-split') - if (existsSync(splitDir)) { - assignments = Object.fromEntries(readdirSync(splitDir).map(file => - [file, readFileSync(join(splitDir, file), 'utf8').trim().split('\n').filter(Boolean)])) - write(`${name}-assignments.json`, JSON.stringify(assignments, null, 2)) - } - const summary = { name, exitCode: result.status, counts, events, assignments } - summaries.push(summary) - write('results.json', JSON.stringify(summaries, null, 2)) - console.log(`${name}: exit=${result.status}, endOfYear=${counts[selected]}, eachWeekendOfYear=${counts[other]}`) - assert.deepEqual(counts, { [selected]: expectedCounts[0], [other]: expectedCounts[1] }, - `Unexpected executions; inspect ${name}.log.`) - assert.equal(result.status === 0, shouldPass, `Unexpected exit status; inspect ${name}.log`) - return result -} - -scenario('baseline', process.execPath, vitest, [1, 1], true) -scenario('relative-filter', process.execPath, [...vitest, selected], [1, 1], true) -scenario('absolute-filter', process.execPath, [...vitest, join(fixture, selected)], [1, 1], true) -const parallel = scenario('ddtest-parallel', binary, - [...common, '--min-parallelism', '2', '--max-parallelism', '2'], expectBug ? [1, 2] : [1, 1], true) -assert.match(parallel.stdout + parallel.stderr, /Test files run: 2/) -const assignmentFiles = Object.values(summaries.at(-1).assignments) -assert.equal(assignmentFiles.length, 2) -assert.ok(assignmentFiles.every(files => files.length === 1)) -assert.deepEqual(assignmentFiles.flat().sort(), [selected, other].sort()) -scenario('ddtest-explicit-selection', binary, - [...common, '--min-parallelism', '1', '--max-parallelism', '1', '--', selected], expectBug ? [1, 1] : [1, 0], true) -scenario('ddtest-exclude', binary, - [...common, '--min-parallelism', '1', '--max-parallelism', '1', - '--tests-exclude-pattern', other], expectBug ? [1, 1] : [1, 0], true) -scenario('baseline-create-once', process.execPath, vitest, [1, 1], true, true) -const createOnce = scenario('ddtest-create-once', binary, - [...common, '--min-parallelism', '2', '--max-parallelism', '2'], expectBug ? [1, 2] : [1, 1], !expectBug, true) -if (expectBug) assert.match(createOnce.stdout + createOnce.stderr, /EEXIST/) -console.log(expectBug - ? 'Confirmed: disjoint assignments expand into duplicate execution; create-once test fails only under ddtest.' - : 'Verified: exact assignments, explicit selection, exclusion, and create-once tests all pass.') -console.log(`Evidence retained in ${fixture}; remove that directory when finished.`) From 9ff19d27095731898213c22d62f6cb98638e53f5 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 17:51:04 +0200 Subject: [PATCH 03/10] Use Vitest specification APIs for exact file execution --- .github/workflows/ci.yml | 5 +- docs/running.md | 4 +- internal/compatibility/vitest_test.go | 223 +++++++++++++++++- .../framework/scripts/vitest_exact_files.mjs | 65 +++-- 4 files changed, 275 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d964b63..4d5e0f39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -270,9 +270,10 @@ jobs: vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" # npm 10.9.8 crashes while resolving Vitest's peer dependencies. # Keep peer resolution enabled: Vitest 5 requires Vite as a peer. - npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" + npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" "dd-trace@5.111.0" DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ - go test -v ./internal/compatibility -run '^TestVitestAdapterIntegration$' + DDTEST_DD_TRACE_NODE_MODULES="${vitest_dir}/node_modules" \ + go test -v ./internal/compatibility -run '^TestVitest(Adapter|Tracing)Integration$' mocha-compatibility: runs-on: ubuntu-latest diff --git a/docs/running.md b/docs/running.md index 537a89e4..bd45ae51 100644 --- a/docs/running.md +++ b/docs/running.md @@ -203,7 +203,9 @@ ddtest run --platform javascript --framework mocha --command "pnpm exec mocha -- For JavaScript/Vitest, the command must invoke Vitest directly. During planning, DDTest uses `list --filesOnly --json` on Vitest 2.0 and newer and the config-aware -discovery API on Vitest 1.6. It appends selected files during execution: +discovery API on Vitest 1.6. During execution, it selects assigned files by exact +canonical path through Vitest's Node API, preserving the command's configuration, +project filters, and reporters: ```bash ddtest run --platform javascript --framework vitest --command "pnpm exec vitest run --project unit*" diff --git a/internal/compatibility/vitest_test.go b/internal/compatibility/vitest_test.go index 662efee8..77514352 100644 --- a/internal/compatibility/vitest_test.go +++ b/internal/compatibility/vitest_test.go @@ -1,14 +1,23 @@ package compatibility import ( + "bytes" "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" "os" "path/filepath" + "slices" + "strconv" "strings" + "sync" "testing" "time" "github.com/DataDog/ddtest/internal/discovery" + "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/framework" ) @@ -20,14 +29,29 @@ func TestVitestAdapterIntegration(t *testing.T) { if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { t.Fatal(err) } - writeFixture(t, root, "vitest.unit.mjs", `export default { + writeFixture(t, root, "vitest.unit.mjs", `import { appendFileSync } from 'node:fs' +if (process.env.DDTEST_VITEST_CONFIG_EVENTS) + appendFileSync(process.env.DDTEST_VITEST_CONFIG_EVENTS, 'config\n') +export default { test: { include: ['checks/**/*.check.js'], setupFiles: ['./setup.js'], + globalSetup: ['./global-setup.js'], }, } + `) writeFixture(t, root, "setup.js", "globalThis.ddtestVitestSetup = true\n") + writeFixture(t, root, "global-setup.js", `import { appendFileSync } from 'node:fs' +export default function () { + if (process.env.DDTEST_VITEST_CONFIG_EVENTS) + appendFileSync(process.env.DDTEST_VITEST_CONFIG_EVENTS, 'setup\n') + return () => { + if (process.env.DDTEST_VITEST_CONFIG_EVENTS) + appendFileSync(process.env.DDTEST_VITEST_CONFIG_EVENTS, 'teardown\n') + } +} +`) writeFixture(t, root, "checks/selected.check.js", `import { expect, test } from 'vitest' test('preserves config while running an assigned file', () => { @@ -56,9 +80,15 @@ test('must not run', () => { wantFiles := []string{"checks/selected.check.js", "checks/unselected.check.js"} requireFiles(t, files, wantFiles) - if err := vitest.RunTests(ctx, []string{"checks/selected.check.js"}, map[string]string{"DDTEST_VITEST_WORKER": "selected"}); err != nil { + lifecycle := filepath.Join(root, "lifecycle.txt") + if err := vitest.RunTests(ctx, []string{"checks/selected.check.js"}, map[string]string{ + "DDTEST_VITEST_WORKER": "selected", "DDTEST_VITEST_CONFIG_EVENTS": lifecycle, + }); err != nil { t.Fatalf("selected-file run failed: %v", err) } + if contents, err := os.ReadFile(lifecycle); err != nil || string(contents) != "config\nsetup\nteardown\n" { + t.Fatalf("Vitest lifecycle = %q, error = %v", contents, err) + } t.Run("exact file membership with overlapping names", func(t *testing.T) { writeFixture(t, root, "vitest.overlap.mjs", `export default { @@ -115,4 +145,193 @@ test('runs only in its assigned batch', () => { t.Fatalf("expected assigned-test failure, got %v", err) } }) + + t.Run("project specifications and configured reporters", func(t *testing.T) { + writeFixture(t, root, "multi-project.config.mjs", `import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) +const major = Number(require('vitest/package.json').version.split('.')[0]) +export default { + test: { + ...(major < 4 ? { workspace: './workspace.mjs' } : { projects: ['./one/vitest.config.mjs', './two/vitest.config.mjs'] }), + reporters: ['json'], + outputFile: './results.json', + }, +} +`) + writeFixture(t, root, "workspace.mjs", `export default ['./one/vitest.config.mjs', './two/vitest.config.mjs']`) + for _, name := range []string{"one", "two"} { + writeFixture(t, root, name+"/vitest.config.mjs", `export default { test: { + name: '`+name+`', root: new URL('..', import.meta.url).pathname, include: ['project-checks/*.test.js'], + env: { DDTEST_PROJECT: '`+name+`' }, +} } +`) + } + writeFixture(t, root, "project-checks/selected.test.js", `import { test } from 'vitest' +import { appendFileSync } from 'node:fs' +test('selected in each project', () => { + appendFileSync(process.env.DDTEST_VITEST_EVENTS, process.env.DDTEST_PROJECT + '\n') +}) +test('filtered by test name', () => { throw new Error('test name filter lost') }) +`) + writeFixture(t, root, "project-checks/unselected.test.js", `import { test } from 'vitest' +test('must not run', () => { throw new Error('file assignment lost') }) +`) + for _, project := range []string{"one", ""} { + events := filepath.Join(root, "projects.txt") + if err := os.WriteFile(events, nil, 0600); err != nil { + t.Fatal(err) + } + args := []string{"--config", "multi-project.config.mjs", "--testNamePattern", "selected in each project"} + want := []string{"one"} + if project != "" { + args = append(args, "--project", project) + } else { + want = append(want, "two") + } + configureFramework(shellCommand(append([]string{filepath.Join(root, "node_modules/.bin/vitest")}, args...)...), "") + if err := framework.NewVitest().RunTests(ctx, []string{"project-checks/selected.test.js"}, map[string]string{"DDTEST_VITEST_EVENTS": events}); err != nil { + t.Fatal(err) + } + contents, err := os.ReadFile(events) + if err != nil { + t.Fatal(err) + } + got := strings.Fields(string(contents)) + slices.Sort(got) + if !slices.Equal(got, want) { + t.Fatalf("executed projects %v, want %v", got, want) + } + report, err := os.ReadFile(filepath.Join(root, "results.json")) + if err != nil { + t.Fatal(err) + } + var result struct { + TestResults []struct{ AssertionResults []struct{ Status string } } + } + if err := json.Unmarshal(report, &result); err != nil { + t.Fatalf("reporter result = %s, error = %v", report, err) + } + passed := 0 + for _, file := range result.TestResults { + for _, assertion := range file.AssertionResults { + if assertion.Status == "passed" { + passed++ + } + } + } + if passed != len(want) { + t.Fatalf("reporter recorded %d passing tests, want %d", passed, len(want)) + } + } + }) + + t.Run("no matching specification", func(t *testing.T) { + configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.unit.mjs"), "") + err := framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, nil) + if err == nil || !strings.Contains(err.Error(), "exit status 1") { + t.Fatalf("expected no-test failure, got %v", err) + } + configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.unit.mjs", "--passWithNoTests"), "") + if err := framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, nil); err != nil { + t.Fatalf("passWithNoTests failed: %v", err) + } + }) +} + +func TestVitestTracingIntegration(t *testing.T) { + nodeModules := requireEnv(t, "DDTEST_VITEST_NODE_MODULES") + tracerModules := requireEnv(t, "DDTEST_DD_TRACE_NODE_MODULES") + resetSettingsAfterTest(t) + root := t.TempDir() + if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { + t.Fatal(err) + } + writeFixture(t, root, "vitest.config.mjs", `export default { test: { dir: 'src', include: ['**/test.js'] } }`) + writeFixture(t, root, "src/endOfYear/test.js", `import { test } from 'vitest' +test('ddtest exact traced assignment', () => {}) +`) + writeFixture(t, root, "src/eachWeekendOfYear/test.js", `import { test } from 'vitest' +test('ddtest unassigned traced file', () => { throw new Error('unassigned file ran') }) +`) + + // Capture actual CI events locally, including session completion and worker + // spans. A passing test run alone would not catch lost Node API tracing. + var mu sync.Mutex + var payloads []byte + agent := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/info" { + _, _ = io.WriteString(w, `{"endpoints":["/evp_proxy/v2"]}`) + return + } + if strings.HasSuffix(r.URL.Path, "/citestcycle") { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + mu.Lock() + payloads = append(payloads, body...) + mu.Unlock() + } + _, _ = io.WriteString(w, `{}`) + })) + defer agent.Close() + t.Chdir(root) + t.Setenv("NODE_OPTIONS", "") + configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest")), "") + vitest := framework.NewVitest() + tracerRoot := filepath.Join(tracerModules, "dd-trace") + vitest.SetPlatformEnv(map[string]string{ + "NODE_OPTIONS": "--require " + strconv.Quote(filepath.Join(tracerRoot, "ci/init.js")) + + " --import " + strconv.Quote(filepath.Join(tracerRoot, "register.js")), + "DD_TRACE_AGENT_URL": agent.URL, + "DD_TRACE_ENABLED": "true", + "DD_CIVISIBILITY_ENABLED": "true", + "DD_CIVISIBILITY_AGENTLESS_ENABLED": "false", + "DD_CIVISIBILITY_ITR_ENABLED": "false", + "DD_CIVISIBILITY_CODE_COVERAGE_ENABLED": "false", + "DD_GIT_METADATA_ENABLED": "false", + "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "false", + "DD_TRACE_STARTUP_LOGS": "false", + }) + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + // Compare against the CLI baseline: the minimum tracer has no Vitest 5 + // sequencer hook, so that combination cannot emit CI events even without + // ddtest. Exercise its file-selection adapter in the other integration test. + if err := (&ext.DefaultCommandExecutor{}).Run(ctx, filepath.Join(root, "node_modules/.bin/vitest"), + []string{"run", "src/endOfYear/test.js", "--exclude", "**/eachWeekendOfYear/**"}, vitest.GetPlatformEnv()); err != nil { + t.Fatal(err) + } + mu.Lock() + baselineTraced := bytes.Contains(payloads, []byte("test_session_end")) + payloads = nil + mu.Unlock() + if !baselineTraced { + manifest, err := os.ReadFile(filepath.Join(nodeModules, "vitest/package.json")) + if err != nil { + t.Fatal(err) + } + var pkg struct{ Version string } + if err := json.Unmarshal(manifest, &pkg); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(pkg.Version, "5.") { + t.Fatal("unmodified Vitest CLI did not emit a CI session") + } + t.Skip("installed dd-trace does not emit a CI session for the unmodified Vitest CLI") + } + if err := vitest.RunTests(ctx, []string{"src/endOfYear/test.js"}, nil); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + for _, marker := range []string{"test_session_end", "test_suite_end", "ddtest exact traced assignment"} { + if !bytes.Contains(payloads, []byte(marker)) { + t.Errorf("CI payloads missing %q", marker) + } + } + if bytes.Contains(payloads, []byte("ddtest unassigned traced file")) { + t.Error("CI payload contains an unassigned test") + } } diff --git a/internal/framework/scripts/vitest_exact_files.mjs b/internal/framework/scripts/vitest_exact_files.mjs index 703f4b5f..e65daed3 100644 --- a/internal/framework/scripts/vitest_exact_files.mjs +++ b/internal/framework/scripts/vitest_exact_files.mjs @@ -1,6 +1,6 @@ import { readFileSync, realpathSync } from 'node:fs' import { createRequire } from 'node:module' -import { basename, dirname, resolve } from 'node:path' +import { basename, resolve } from 'node:path' import { pathToFileURL } from 'node:url' const entrypoint = process.argv[1] || '' @@ -13,25 +13,56 @@ if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypo const canonicalPath = file => realpathSync(resolve(file)) const selected = new Set(JSON.parse(readFileSync(filesPath, 'utf8')).map(canonicalPath)) const require = createRequire(pathToFileURL(realpathSync(entrypoint))) - const { createVitest } = await import(pathToFileURL(require.resolve('vitest/node')).href) - - // Vitest does not export the project class on every supported version. An - // empty, config-free context gives us its prototype without loading the - // user's configuration twice. The original CLI still owns the real run, - // including reporters, coverage, tracer hooks, exit codes, and teardown. - const context = await createVitest('test', { - root: dirname(filesPath), config: false, watch: false, include: [], - }) + const { startVitest, parseCLI } = await import(pathToFileURL(require.resolve('vitest/node')).href) + const major = Number(require('vitest/package.json').version.split('.')[0]) + // Use Vitest's own parser to preserve command options. Positional filters + // are replaced by the exact assignment. + const { options } = parseCLI(['vitest', ...process.argv.slice(2)]) + if (options.exclude) { + options.cliExclude = options.exclude + delete options.exclude + } + // startVitest performs package validation and starts the Datadog session. + // Standalone watch mode keeps its context open for the explicit API run; + // disable Vite's watcher so this remains a single ddtest worker invocation. + const config = { + ...options, run: false, watch: true, standalone: true, + } + const overrides = { server: { watch: null } } + const context = major >= 5 + ? await startVitest([], config, overrides) + : await startVitest('test', [], config, overrides) + if (!context) { + throw new Error('Vitest could not initialize the exact-file runner') + } try { - const project = context.projects[0] - if (!project || typeof project.filterFiles !== 'function') { - throw new Error('ddtest cannot enforce exact file selection with this Vitest version') + if (process.exitCode) { + throw new Error('Vitest failed to initialize the exact-file runner') } - const prototype = Object.getPrototypeOf(project) - prototype.filterFiles = function (files) { - return files.filter(file => selected.has(canonicalPath(file))) + const modern = major >= 3 + // Vitest 3+ has a public specification API. Vitest 1-2 use the older + // discovery/run methods; keep the original specs (including project/pool). + let specs = modern + ? await context.getRelevantTestSpecifications() + : await context.filterTestsBySource(await context.globTestFiles()) + specs = specs.filter(spec => selected.has(canonicalPath(spec.moduleId ?? spec[1]))) + if (specs.length === 0) { + console.error('No assigned Vitest test files found') + process.exitCode = context.config.passWithNoTests ? 0 : 1 + } else if (modern) { + await context.runTestSpecifications(specs, true) + } else { + await context.runFiles(specs, true) } + } catch (error) { + console.error(error) + process.exitCode = 1 } finally { - await context.close() + await context.exit() } + // The API run replaces the CLI entrypoint. Flush reporters before exiting so + // the original CLI cannot start a second run with substring filters. + await Promise.all([process.stdout, process.stderr].map(stream => + new Promise(resolve => stream.write('', resolve)))) + process.exit() } From e77a97f118dcb52fbf2c290d75c45e7cc288f217 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 18:07:04 +0200 Subject: [PATCH 04/10] Separate modern and legacy Vitest execution adapters --- .../framework/scripts/vitest_exact_files.mjs | 16 ++++++---------- .../scripts/vitest_exact_files_legacy.mjs | 13 +++++++++++++ .../scripts/vitest_exact_files_modern.mjs | 12 ++++++++++++ internal/framework/vitest.go | 13 ++++++++++++- internal/framework/vitest_test.go | 6 ++++-- 5 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 internal/framework/scripts/vitest_exact_files_legacy.mjs create mode 100644 internal/framework/scripts/vitest_exact_files_modern.mjs diff --git a/internal/framework/scripts/vitest_exact_files.mjs b/internal/framework/scripts/vitest_exact_files.mjs index e65daed3..78afb6df 100644 --- a/internal/framework/scripts/vitest_exact_files.mjs +++ b/internal/framework/scripts/vitest_exact_files.mjs @@ -15,6 +15,9 @@ if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypo const require = createRequire(pathToFileURL(realpathSync(entrypoint))) const { startVitest, parseCLI } = await import(pathToFileURL(require.resolve('vitest/node')).href) const major = Number(require('vitest/package.json').version.split('.')[0]) + const runner = major >= 3 + ? await import('./vitest_exact_files_modern.mjs') + : await import('./vitest_exact_files_legacy.mjs') // Use Vitest's own parser to preserve command options. Positional filters // are replaced by the exact assignment. const { options } = parseCLI(['vitest', ...process.argv.slice(2)]) @@ -39,20 +42,13 @@ if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypo if (process.exitCode) { throw new Error('Vitest failed to initialize the exact-file runner') } - const modern = major >= 3 - // Vitest 3+ has a public specification API. Vitest 1-2 use the older - // discovery/run methods; keep the original specs (including project/pool). - let specs = modern - ? await context.getRelevantTestSpecifications() - : await context.filterTestsBySource(await context.globTestFiles()) - specs = specs.filter(spec => selected.has(canonicalPath(spec.moduleId ?? spec[1]))) + const discovered = await runner.discoverSpecifications(context) + const specs = discovered.filter(spec => selected.has(canonicalPath(runner.filePath(spec)))) if (specs.length === 0) { console.error('No assigned Vitest test files found') process.exitCode = context.config.passWithNoTests ? 0 : 1 - } else if (modern) { - await context.runTestSpecifications(specs, true) } else { - await context.runFiles(specs, true) + await runner.runSpecifications(context, specs) } } catch (error) { console.error(error) diff --git a/internal/framework/scripts/vitest_exact_files_legacy.mjs b/internal/framework/scripts/vitest_exact_files_legacy.mjs new file mode 100644 index 00000000..78c0f12b --- /dev/null +++ b/internal/framework/scripts/vitest_exact_files_legacy.mjs @@ -0,0 +1,13 @@ +// Vitest 1.6-2 compatibility only: these APIs predate the public specification +// API. Vitest 1 returns workspace tuples; Vitest 2 returns specification objects. +export async function discoverSpecifications(context) { + return context.filterTestsBySource(await context.globTestFiles()) +} + +export function filePath(specification) { + return specification.moduleId ?? specification[1] +} + +export function runSpecifications(context, specifications) { + return context.runFiles(specifications, true) +} diff --git a/internal/framework/scripts/vitest_exact_files_modern.mjs b/internal/framework/scripts/vitest_exact_files_modern.mjs new file mode 100644 index 00000000..c9c9e787 --- /dev/null +++ b/internal/framework/scripts/vitest_exact_files_modern.mjs @@ -0,0 +1,12 @@ +// Vitest 3+ public specification API. Keep project/pool specifications intact. +export function discoverSpecifications(context) { + return context.getRelevantTestSpecifications() +} + +export function filePath(specification) { + return specification.moduleId +} + +export function runSpecifications(context, specifications) { + return context.runTestSpecifications(specifications, true) +} diff --git a/internal/framework/vitest.go b/internal/framework/vitest.go index 74bbfb67..213f7756 100644 --- a/internal/framework/vitest.go +++ b/internal/framework/vitest.go @@ -35,6 +35,12 @@ var vitestV1DiscoveryScript string //go:embed scripts/vitest_exact_files.mjs var vitestExactFilesScript string +//go:embed scripts/vitest_exact_files_modern.mjs +var vitestExactFilesModernScript string + +//go:embed scripts/vitest_exact_files_legacy.mjs +var vitestExactFilesLegacyScript string + var vitestTestFileExtensions = []string{"js", "jsx", "ts", "tsx", "mjs", "mts", "cjs", "cts"} type vitestExecutor interface { @@ -236,7 +242,12 @@ func prepareVitestExactFiles(baseEnv map[string]string, testFiles []string) (str } adapterPath := filepath.Join(adapterDir, "exact-files.mjs") filesPath := filepath.Join(adapterDir, "files.json") - for path, contents := range map[string][]byte{adapterPath: []byte(vitestExactFilesScript), filesPath: encodedFiles} { + for path, contents := range map[string][]byte{ + adapterPath: []byte(vitestExactFilesScript), + filepath.Join(adapterDir, "vitest_exact_files_modern.mjs"): []byte(vitestExactFilesModernScript), + filepath.Join(adapterDir, "vitest_exact_files_legacy.mjs"): []byte(vitestExactFilesLegacyScript), + filesPath: encodedFiles, + } { if err := os.WriteFile(path, contents, 0600); err != nil { _ = os.RemoveAll(adapterDir) return "", nil, fmt.Errorf("failed to write Vitest execution adapter: %w", err) diff --git a/internal/framework/vitest_test.go b/internal/framework/vitest_test.go index 5a1a9ac1..74b3afbc 100644 --- a/internal/framework/vitest_test.go +++ b/internal/framework/vitest_test.go @@ -502,8 +502,10 @@ func TestVitest_RunTests_ExactSelectionEnvironmentAndCleanup(t *testing.T) { if !strings.HasPrefix(env["NODE_OPTIONS"], workerEnv["NODE_OPTIONS"]+" --import ") { t.Fatalf("lost worker NODE_OPTIONS: %q", env["NODE_OPTIONS"]) } - if _, err := os.Stat(filepath.Join(filepath.Dir(selectionPath), "exact-files.mjs")); err != nil { - t.Fatal(err) + for _, name := range []string{"exact-files.mjs", "vitest_exact_files_modern.mjs", "vitest_exact_files_legacy.mjs"} { + if _, err := os.Stat(filepath.Join(filepath.Dir(selectionPath), name)); err != nil { + t.Fatal(err) + } } return runErr }} From 728ac2d5e93abf1bf792298d0d5c44db396b840a Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 18:11:41 +0200 Subject: [PATCH 05/10] Clarify Vitest exact-file adapter preparation --- internal/framework/vitest.go | 94 ++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 25 deletions(-) diff --git a/internal/framework/vitest.go b/internal/framework/vitest.go index 213f7756..31783567 100644 --- a/internal/framework/vitest.go +++ b/internal/framework/vitest.go @@ -223,46 +223,90 @@ func (v *Vitest) RunTests(ctx context.Context, testFiles []string, envMap map[st return v.executor.Run(ctx, command, args, adapterEnv) } +// prepareVitestExactFiles writes a worker's adapter and assignment, then builds +// the environment that loads them. The caller must remove adapterDir after the run. func prepareVitestExactFiles(baseEnv map[string]string, testFiles []string) (string, map[string]string, error) { - selectedFiles := make([]string, len(testFiles)) + assignedFilesJSON, err := encodeVitestAssignedFiles(testFiles) + if err != nil { + return "", nil, err + } + + adapterDir, err := os.MkdirTemp("", "ddtest-vitest-run-*") + if err != nil { + return "", nil, fmt.Errorf("failed to create Vitest execution adapter: %w", err) + } + if err := writeVitestExecutionAdapter(adapterDir, assignedFilesJSON); err != nil { + _ = os.RemoveAll(adapterDir) + return "", nil, err + } + + return adapterDir, vitestExactFilesEnv(baseEnv, adapterDir), nil +} + +func encodeVitestAssignedFiles(testFiles []string) ([]byte, error) { + // Resolve paths in ddtest's working directory before any command wrapper starts. + // The Node adapter resolves symlinks when comparing these paths with Vitest's specs. + absoluteFiles := make([]string, len(testFiles)) for i, file := range testFiles { absolute, err := filepath.Abs(file) if err != nil { - return "", nil, fmt.Errorf("failed to resolve Vitest test file: %w", err) + return nil, fmt.Errorf("failed to resolve Vitest test file: %w", err) } - selectedFiles[i] = absolute + absoluteFiles[i] = absolute } - encodedFiles, err := json.Marshal(selectedFiles) + encodedFiles, err := json.Marshal(absoluteFiles) if err != nil { - return "", nil, fmt.Errorf("failed to encode Vitest test files: %w", err) + return nil, fmt.Errorf("failed to encode Vitest test files: %w", err) } - adapterDir, err := os.MkdirTemp("", "ddtest-vitest-run-*") - if err != nil { - return "", nil, fmt.Errorf("failed to create Vitest execution adapter: %w", err) - } - adapterPath := filepath.Join(adapterDir, "exact-files.mjs") - filesPath := filepath.Join(adapterDir, "files.json") - for path, contents := range map[string][]byte{ - adapterPath: []byte(vitestExactFilesScript), - filepath.Join(adapterDir, "vitest_exact_files_modern.mjs"): []byte(vitestExactFilesModernScript), - filepath.Join(adapterDir, "vitest_exact_files_legacy.mjs"): []byte(vitestExactFilesLegacyScript), - filesPath: encodedFiles, - } { - if err := os.WriteFile(path, contents, 0600); err != nil { - _ = os.RemoveAll(adapterDir) - return "", nil, fmt.Errorf("failed to write Vitest execution adapter: %w", err) + return encodedFiles, nil +} + +func writeVitestExecutionAdapter(adapterDir string, assignedFilesJSON []byte) error { + // Keep the modules together so the entrypoint's relative imports work. + // JSON keeps the assignment independent of command-line quoting. + files := []struct { + name string + contents []byte + }{ + {"exact-files.mjs", []byte(vitestExactFilesScript)}, + {"vitest_exact_files_modern.mjs", []byte(vitestExactFilesModernScript)}, + {"vitest_exact_files_legacy.mjs", []byte(vitestExactFilesLegacyScript)}, + {"files.json", assignedFilesJSON}, + } + for _, file := range files { + path := filepath.Join(adapterDir, file.name) + if err := os.WriteFile(path, file.contents, 0600); err != nil { + return fmt.Errorf("failed to write Vitest execution adapter file %q: %w", file.name, err) } } + return nil +} + +func vitestExactFilesEnv(baseEnv map[string]string, adapterDir string) map[string]string { adapterEnv := make(map[string]string, len(baseEnv)+2) maps.Copy(adapterEnv, baseEnv) - nodeOptions, ok := adapterEnv[nodeOptionsEnvVar] + + // Preserve worker options, including tracing hooks. Inherit the process options + // only when the worker hasn't set NODE_OPTIONS (an explicit empty value wins). + nodeOptions, ok := baseEnv[nodeOptionsEnvVar] if !ok { nodeOptions = os.Getenv(nodeOptionsEnvVar) } - importURL := &url.URL{Scheme: "file", Path: "/" + strings.TrimPrefix(filepath.ToSlash(adapterPath), "/")} - adapterEnv[nodeOptionsEnvVar] = strings.TrimSpace(nodeOptions + " --import " + strconv.Quote(importURL.String())) - adapterEnv[vitestSelectedFilesEnv] = filesPath - return adapterDir, adapterEnv, nil + + // NODE_OPTIONS also reaches Vitest launched through npx/pnpm. The entrypoint + // activates only in the Vitest process and reads its assignment from files.json. + adapterPath := filepath.Join(adapterDir, "exact-files.mjs") + // File URLs need forward slashes and a leading slash, including on Windows. + importPath := filepath.ToSlash(adapterPath) + if !strings.HasPrefix(importPath, "/") { + importPath = "/" + importPath + } + importURL := url.URL{Scheme: "file", Path: importPath} + // Quote the URL so NODE_OPTIONS also works when the temp path contains spaces. + importOption := "--import " + strconv.Quote(importURL.String()) + adapterEnv[nodeOptionsEnvVar] = strings.TrimSpace(nodeOptions + " " + importOption) + adapterEnv[vitestSelectedFilesEnv] = filepath.Join(adapterDir, "files.json") + return adapterEnv } func (v *Vitest) discoveryEnv() map[string]string { From cbf776effd9e5242476b4d3b3cb4e59c7969f4d7 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 19:10:24 +0200 Subject: [PATCH 06/10] Use createVitest for exact-file worker lifecycle --- .github/workflows/ci.yml | 2 +- README.md | 2 +- internal/compatibility/vitest_test.go | 131 ++++++++++++------ .../framework/scripts/vitest_exact_files.mjs | 31 ++--- .../scripts/vitest_exact_files_legacy.mjs | 4 + .../scripts/vitest_exact_files_modern.mjs | 5 + 6 files changed, 117 insertions(+), 58 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d5e0f39..781831b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -270,7 +270,7 @@ jobs: vitest_dir="${RUNNER_TEMP}/vitest-${{ matrix.vitest }}" # npm 10.9.8 crashes while resolving Vitest's peer dependencies. # Keep peer resolution enabled: Vitest 5 requires Vite as a peer. - npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" "dd-trace@5.111.0" + npx --yes npm@11.11.1 install --prefix "${vitest_dir}" "vitest@${{ matrix.vitest }}" "@vitest/coverage-v8@${{ matrix.vitest }}" "dd-trace@5.125.0" DDTEST_VITEST_NODE_MODULES="${vitest_dir}/node_modules" \ DDTEST_DD_TRACE_NODE_MODULES="${vitest_dir}/node_modules" \ go test -v ./internal/compatibility -run '^TestVitest(Adapter|Tracing)Integration$' diff --git a/README.md b/README.md index 4b193f7a..ff5539e6 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Minimum supported library and runtime requirements: Cucumber support is tested with `@cucumber/cucumber` 7 through 13; Cypress support requires Cypress 12 or higher; Mocha support requires Mocha 8 or higher; Playwright support requires Playwright 1.18 or higher; Vitest support requires - Vitest 1.6 or higher. + Vitest 1.6 or higher and `dd-trace` **5.125.0** or higher for its programmatic runner. For instructions on setting up Test Optimization, see the [Datadog Test Optimization documentation](https://docs.datadoghq.com/tests/setup/). diff --git a/internal/compatibility/vitest_test.go b/internal/compatibility/vitest_test.go index 77514352..0cc05af9 100644 --- a/internal/compatibility/vitest_test.go +++ b/internal/compatibility/vitest_test.go @@ -17,7 +17,6 @@ import ( "time" "github.com/DataDog/ddtest/internal/discovery" - "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/framework" ) @@ -56,6 +55,8 @@ export default function () { test('preserves config while running an assigned file', () => { expect(globalThis.ddtestVitestSetup).toBe(true) + expect(process.env.TEST).toBe('true') + expect(process.env.VITEST).toBe('true') expect(process.env.DDTEST_VITEST_WORKER).toBe('selected') }) `) @@ -108,6 +109,7 @@ test('runs only in its assigned batch', () => { for _, tc := range []struct { name, command, selected string }{ + {"shard in run mode", shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.overlap.mjs", "--shard=1/1"), "src/endOfYear/test.js"}, {"direct", shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.overlap.mjs"), "src/endOfYear/test.js"}, {"node with separator", shellCommand("node", filepath.Join(root, "node_modules/vitest/vitest.mjs"), "run", "--config", "vitest.overlap.mjs", "--", "old.test.js"), filepath.Join(root, "src/endOfYear/test.js")}, {"package manager", shellCommand("npx", "--no-install", "vitest", "run", "--config", "vitest.overlap.mjs"), "src/eachWeekendOfYear/test.js"}, @@ -225,6 +227,66 @@ test('must not run', () => { throw new Error('file assignment lost') }) } }) + t.Run("CI snapshots require an explicit update", func(t *testing.T) { + writeFixture(t, root, "snapshot.config.mjs", `export default { test: { include: ['snapshot.test.js'] } }`) + writeFixture(t, root, "snapshot.test.js", `import { expect, test } from 'vitest' +test('snapshot policy', () => { expect({ assigned: true }).toMatchSnapshot() }) +`) + command := []string{filepath.Join(root, "node_modules/.bin/vitest"), "--config", "snapshot.config.mjs"} + configureFramework(shellCommand(command...), "") + err := framework.NewVitest().RunTests(ctx, []string{"snapshot.test.js"}, map[string]string{"CI": "true"}) + if err == nil || !strings.Contains(err.Error(), "exit status 1") { + t.Fatalf("expected missing-snapshot failure in CI, got %v", err) + } + snapshot := filepath.Join(root, "__snapshots__/snapshot.test.js.snap") + if _, err := os.Stat(snapshot); !os.IsNotExist(err) { + t.Fatalf("snapshot should not be written without --update, got %v", err) + } + configureFramework(shellCommand(append(command, "--update")...), "") + if err := framework.NewVitest().RunTests(ctx, []string{"snapshot.test.js"}, map[string]string{"CI": "true"}); err != nil { + t.Fatalf("explicit snapshot update failed: %v", err) + } + if _, err := os.Stat(snapshot); err != nil { + t.Fatalf("explicit update did not write a snapshot: %v", err) + } + }) + + t.Run("coverage provider and reporting", func(t *testing.T) { + writeFixture(t, root, "coverage.config.mjs", `export default { test: { + include: ['coverage.test.js'], + coverage: { enabled: true, provider: 'v8', include: ['math.js'], reporter: ['json'], reportsDirectory: './coverage-report' }, +} }`) + writeFixture(t, root, "math.js", `export function add(a, b) { return a + b }`) + writeFixture(t, root, "coverage.test.js", `import { expect, test } from 'vitest' +import { add } from './math.js' +test('records coverage', () => { expect(add(1, 2)).toBe(3) }) +`) + configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "coverage.config.mjs"), "") + if err := framework.NewVitest().RunTests(ctx, []string{"coverage.test.js"}, nil); err != nil { + t.Fatal(err) + } + report, err := os.ReadFile(filepath.Join(root, "coverage-report/coverage-final.json")) + if err != nil { + t.Fatal(err) + } + var coverage map[string]struct { + Statements map[string]int `json:"s"` + } + if err := json.Unmarshal(report, &coverage); err != nil { + t.Fatal(err) + } + for file, counts := range coverage { + if filepath.Base(file) == "math.js" { + for _, count := range counts.Statements { + if count > 0 { + return + } + } + } + } + t.Fatalf("coverage did not record executed statements in math.js: %s", report) + }) + t.Run("no matching specification", func(t *testing.T) { configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.unit.mjs"), "") err := framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, nil) @@ -248,7 +310,9 @@ func TestVitestTracingIntegration(t *testing.T) { } writeFixture(t, root, "vitest.config.mjs", `export default { test: { dir: 'src', include: ['**/test.js'] } }`) writeFixture(t, root, "src/endOfYear/test.js", `import { test } from 'vitest' -test('ddtest exact traced assignment', () => {}) +test('ddtest exact traced assignment', () => { + if (process.env.DDTEST_FAIL_ASSIGNED === 'true') throw new Error('assigned failure') +}) `) writeFixture(t, root, "src/eachWeekendOfYear/test.js", `import { test } from 'vitest' test('ddtest unassigned traced file', () => { throw new Error('unassigned file ran') }) @@ -296,42 +360,31 @@ test('ddtest unassigned traced file', () => { throw new Error('unassigned file r }) ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) defer cancel() - // Compare against the CLI baseline: the minimum tracer has no Vitest 5 - // sequencer hook, so that combination cannot emit CI events even without - // ddtest. Exercise its file-selection adapter in the other integration test. - if err := (&ext.DefaultCommandExecutor{}).Run(ctx, filepath.Join(root, "node_modules/.bin/vitest"), - []string{"run", "src/endOfYear/test.js", "--exclude", "**/eachWeekendOfYear/**"}, vitest.GetPlatformEnv()); err != nil { - t.Fatal(err) - } - mu.Lock() - baselineTraced := bytes.Contains(payloads, []byte("test_session_end")) - payloads = nil - mu.Unlock() - if !baselineTraced { - manifest, err := os.ReadFile(filepath.Join(nodeModules, "vitest/package.json")) - if err != nil { - t.Fatal(err) - } - var pkg struct{ Version string } - if err := json.Unmarshal(manifest, &pkg); err != nil { - t.Fatal(err) - } - if !strings.HasPrefix(pkg.Version, "5.") { - t.Fatal("unmodified Vitest CLI did not emit a CI session") - } - t.Skip("installed dd-trace does not emit a CI session for the unmodified Vitest CLI") - } - if err := vitest.RunTests(ctx, []string{"src/endOfYear/test.js"}, nil); err != nil { - t.Fatal(err) - } - mu.Lock() - defer mu.Unlock() - for _, marker := range []string{"test_session_end", "test_suite_end", "ddtest exact traced assignment"} { - if !bytes.Contains(payloads, []byte(marker)) { - t.Errorf("CI payloads missing %q", marker) - } - } - if bytes.Contains(payloads, []byte("ddtest unassigned traced file")) { - t.Error("CI payload contains an unassigned test") + for _, failure := range []bool{false, true} { + t.Run("assigned failure="+strconv.FormatBool(failure), func(t *testing.T) { + mu.Lock() + payloads = nil + mu.Unlock() + err := vitest.RunTests(ctx, []string{"src/endOfYear/test.js"}, map[string]string{ + "DDTEST_FAIL_ASSIGNED": strconv.FormatBool(failure), + }) + if failure { + if err == nil || !strings.Contains(err.Error(), "exit status 1") { + t.Fatalf("expected assigned-test failure, got %v", err) + } + } else if err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + for _, marker := range []string{"test_session_end", "test_suite_end", "ddtest exact traced assignment"} { + if !bytes.Contains(payloads, []byte(marker)) { + t.Errorf("CI payloads missing %q", marker) + } + } + if bytes.Contains(payloads, []byte("ddtest unassigned traced file")) { + t.Error("CI payload contains an unassigned test") + } + }) } } diff --git a/internal/framework/scripts/vitest_exact_files.mjs b/internal/framework/scripts/vitest_exact_files.mjs index 78afb6df..3e44b90b 100644 --- a/internal/framework/scripts/vitest_exact_files.mjs +++ b/internal/framework/scripts/vitest_exact_files.mjs @@ -13,7 +13,7 @@ if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypo const canonicalPath = file => realpathSync(resolve(file)) const selected = new Set(JSON.parse(readFileSync(filesPath, 'utf8')).map(canonicalPath)) const require = createRequire(pathToFileURL(realpathSync(entrypoint))) - const { startVitest, parseCLI } = await import(pathToFileURL(require.resolve('vitest/node')).href) + const { createVitest, parseCLI } = await import(pathToFileURL(require.resolve('vitest/node')).href) const major = Number(require('vitest/package.json').version.split('.')[0]) const runner = major >= 3 ? await import('./vitest_exact_files_modern.mjs') @@ -25,23 +25,20 @@ if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypo options.cliExclude = options.exclude delete options.exclude } - // startVitest performs package validation and starts the Datadog session. - // Standalone watch mode keeps its context open for the explicit API run; - // disable Vite's watcher so this remains a single ddtest worker invocation. - const config = { - ...options, run: false, watch: true, standalone: true, - } - const overrides = { server: { watch: null } } + // Match the CLI's environment before loading the user's configuration. + process.env.TEST = 'true' + process.env.VITEST = 'true' + process.env.NODE_ENV ??= 'test' + + // Own the lifecycle so discovery cannot execute files before we filter them. + // dd-trace 5.125+ instruments createVitest directly; no watch mode is needed. + const config = { ...options, run: true, watch: false } const context = major >= 5 - ? await startVitest([], config, overrides) - : await startVitest('test', [], config, overrides) - if (!context) { - throw new Error('Vitest could not initialize the exact-file runner') - } + ? await createVitest(config) + : await createVitest('test', config) try { - if (process.exitCode) { - throw new Error('Vitest failed to initialize the exact-file runner') - } + // Initialize reporters and coverage without starting a test run. + await runner.initialize(context) const discovered = await runner.discoverSpecifications(context) const specs = discovered.filter(spec => selected.has(canonicalPath(runner.filePath(spec)))) if (specs.length === 0) { @@ -54,7 +51,7 @@ if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypo console.error(error) process.exitCode = 1 } finally { - await context.exit() + await context.close() } // The API run replaces the CLI entrypoint. Flush reporters before exiting so // the original CLI cannot start a second run with substring filters. diff --git a/internal/framework/scripts/vitest_exact_files_legacy.mjs b/internal/framework/scripts/vitest_exact_files_legacy.mjs index 78c0f12b..c5f02835 100644 --- a/internal/framework/scripts/vitest_exact_files_legacy.mjs +++ b/internal/framework/scripts/vitest_exact_files_legacy.mjs @@ -1,5 +1,9 @@ // Vitest 1.6-2 compatibility only: these APIs predate the public specification // API. Vitest 1 returns workspace tuples; Vitest 2 returns specification objects. +export function initialize(context) { + return context.init() +} + export async function discoverSpecifications(context) { return context.filterTestsBySource(await context.globTestFiles()) } diff --git a/internal/framework/scripts/vitest_exact_files_modern.mjs b/internal/framework/scripts/vitest_exact_files_modern.mjs index c9c9e787..07e4c095 100644 --- a/internal/framework/scripts/vitest_exact_files_modern.mjs +++ b/internal/framework/scripts/vitest_exact_files_modern.mjs @@ -1,4 +1,9 @@ // Vitest 3+ public specification API. Keep project/pool specifications intact. +export function initialize(context) { + // Vitest 4 renamed init to standalone; both initialize without running tests. + return context.standalone ? context.standalone() : context.init() +} + export function discoverSpecifications(context) { return context.getRelevantTestSpecifications() } From f065814029d924a0fb122987475a7a94498b85b0 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 19:45:08 +0200 Subject: [PATCH 07/10] Run Vitest directly through its Node API --- README.md | 6 +- docs/best_practices.md | 15 +- docs/running.md | 103 +++-- docs/settings.md | 3 +- docs/third-party-runners.md | 14 +- internal/cmd/cmd.go | 4 +- internal/cmd/cmd_test.go | 7 + internal/compatibility/vitest_test.go | 82 ++-- internal/framework/command_override_test.go | 4 +- internal/framework/scripts/vitest.mjs | 44 +++ .../framework/scripts/vitest_exact_files.mjs | 61 --- ...act_files_legacy.mjs => vitest_legacy.mjs} | 0 ...act_files_modern.mjs => vitest_modern.mjs} | 0 .../framework/scripts/vitest_v1_discovery.mjs | 14 - internal/framework/vitest.go | 365 ++++-------------- internal/framework/vitest_test.go | 344 +++++------------ internal/planner/report_test.go | 2 + internal/settings/settings.go | 6 + 18 files changed, 376 insertions(+), 698 deletions(-) create mode 100644 internal/framework/scripts/vitest.mjs delete mode 100644 internal/framework/scripts/vitest_exact_files.mjs rename internal/framework/scripts/{vitest_exact_files_legacy.mjs => vitest_legacy.mjs} (100%) rename internal/framework/scripts/{vitest_exact_files_modern.mjs => vitest_modern.mjs} (100%) delete mode 100644 internal/framework/scripts/vitest_v1_discovery.mjs diff --git a/README.md b/README.md index ff5539e6..2fa38ca5 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ Minimum supported library and runtime requirements: Cucumber support is tested with `@cucumber/cucumber` 7 through 13; Cypress support requires Cypress 12 or higher; Mocha support requires Mocha 8 or higher; Playwright support requires Playwright 1.18 or higher; Vitest support requires - Vitest 1.6 or higher and `dd-trace` **5.125.0** or higher for its programmatic runner. + Vitest 1.6 or higher and `dd-trace` **5.125.0** or higher. Vitest uses a direct + Node API integration; see [configuration and migration instructions](docs/running.md#vitest-integration). For instructions on setting up Test Optimization, see the [Datadog Test Optimization documentation](https://docs.datadoghq.com/tests/setup/). @@ -216,7 +217,8 @@ parallelism details, see [Running DDTest](docs/running.md). | --- | --- | | `--platform` | Language/platform. Currently supported: `ruby`, `python`, `javascript`. | | `--framework` | Test framework. Currently supported: `rspec`, `minitest`, `pytest`, `cucumber`, `cypress`, `jest`, `mocha`, `playwright`, `vitest`. | -| `--command` | Override the default base command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, Playwright, and Vitest run/discovery, and pytest run/discovery (since 1.7.0). For ddtest versions prior to 1.7.0 with pytest, the command cannot be changed. Pass extra flags with `PYTEST_ADDOPTS`. | +| `--command` | Override the default base command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, and Playwright run/discovery, and pytest run/discovery (since 1.7.0). For ddtest versions prior to 1.7.0 with pytest, the command cannot be changed. Pass extra flags with `PYTEST_ADDOPTS`. Vitest rejects this option; use `--vitest-config` and configure options in Vitest. | +| `--vitest-config` | Vitest config file used by both planning and execution; defaults to Vitest config discovery. See [migration instructions](docs/running.md#vitest-integration). | | `--min-parallelism` | Minimum CI node or worker count DDTest considers when planning. | | `--max-parallelism` | Maximum CI node or worker count DDTest considers when planning. | | `--target-time` | Target wall time DDTest tries to satisfy when selecting parallelism. | diff --git a/docs/best_practices.md b/docs/best_practices.md index 63ae0a89..d7a1fc67 100644 --- a/docs/best_practices.md +++ b/docs/best_practices.md @@ -170,17 +170,18 @@ file list and Jest flags itself. ## Vitest Support -Use `--command` when your project runs Vitest through a package manager or uses -Vitest projects: +Install Vitest and `dd-trace` 5.125.0 or higher in the project. Put Vitest options +in its configuration; DDTest uses the Node API directly and rejects `--command`. +Select a non-default configuration consistently during planning and execution: ```bash -ddtest run --platform javascript --framework vitest --command "pnpm exec vitest run --project unit*" +ddtest plan --platform javascript --framework vitest --vitest-config vitest.ci.config.ts +ddtest run --platform javascript --framework vitest --vitest-config vitest.ci.config.ts ``` -The command must invoke Vitest directly. During planning, DDTest changes the -`run` subcommand to `list --filesOnly --json` on Vitest 2.0 and newer. On Vitest 1.6, -DDTest passes the Vitest arguments to its config-aware discovery API. DDTest -supplies the selected test files during execution. +Run package-script setup steps before DDTest and export any required environment +variables. See [Vitest integration](running.md#vitest-integration) for migrating +command flags, selecting projects, and configuring reporters and coverage. ## Mocha Support diff --git a/docs/running.md b/docs/running.md index bd45ae51..74df1bc0 100644 --- a/docs/running.md +++ b/docs/running.md @@ -179,7 +179,7 @@ starting each worker. Use `--command` to override the framework's default base test command where supported. DDTest applies this override to RSpec run and full discovery, Minitest run and full discovery, Cucumber, Cypress, Jest, Mocha, -Playwright, and Vitest run and file discovery, and pytest run and discovery +and Playwright run and file discovery, and pytest run and discovery (since 1.7.0): ```bash @@ -201,15 +201,9 @@ replaces configured `spec` entries with each worker's assigned files: ddtest run --platform javascript --framework mocha --command "pnpm exec mocha --parallel" ``` -For JavaScript/Vitest, the command must invoke Vitest directly. During planning, -DDTest uses `list --filesOnly --json` on Vitest 2.0 and newer and the config-aware -discovery API on Vitest 1.6. During execution, it selects assigned files by exact -canonical path through Vitest's Node API, preserving the command's configuration, -project filters, and reporters: - -```bash -ddtest run --platform javascript --framework vitest --command "pnpm exec vitest run --project unit*" -``` +Vitest uses a direct Node API integration and does not accept `--command`. +Use `--vitest-config` to select a configuration file; see +[Vitest integration](#vitest-integration) for migration instructions. For JavaScript/Cypress, the command must invoke Cypress directly. DDTest keeps configuration options such as `--project`, `--config-file`, `--config`, @@ -360,35 +354,78 @@ expects Mocha to be resolvable from the current project. Discovery removes the Datadog CI preload from `NODE_OPTIONS`; test runs retain it for Test Optimization instrumentation. -## Vitest Discovery And Instrumentation +## Vitest Integration -For JavaScript/Vitest 2.0 or higher, DDTest discovers test files with Vitest's -native `list --filesOnly --json` command. It uses this priority: +DDTest launches its own Node adapter using the project's installed `vitest/node` +API. Install Vitest **1.6 or higher**, `dd-trace` **5.125.0 or higher**, and any +configured coverage or environment packages before running DDTest. Run from the +project directory where Node can resolve those packages. DDTest does not install +Vitest through `npx`. -1. `--command` when set, replacing its Vitest subcommand with `list` and - appending `--filesOnly --json`. -2. The local executable `node_modules/.bin/vitest` when present. -3. `npx vitest`. +Both planning and execution load the normal Vitest/Vite configuration. To use a +non-default file, pass the same `--vitest-config` value to both commands: -Vitest resolves its own Vite/Vitest configuration, projects, and default test -matching. When `--tests-location` or `--tests-exclude-pattern` is set, DDTest -filters the file list returned by Vitest after discovery. - -Vitest 1.6 does not support `list --filesOnly`. When DDTest detects that specific -unsupported-option error, it uses the `vitest/node` discovery API instead. This -loads the project's Vitest configuration and discovers files for its configured -projects, include and exclude patterns, and CLI filters without executing tests. -If that API is unavailable, DDTest falls back to its own filesystem glob using -`--tests-location` or the default Vitest test-file pattern. +```bash +ddtest plan --platform javascript --framework vitest --vitest-config vitest.ci.config.ts +ddtest run --platform javascript --framework vitest --vitest-config vitest.ci.config.ts +``` -DDTest adds the CI require described above and a `--import` for Vitest worker -processes. It preserves an existing Datadog register import; otherwise it uses +Alternatively, set +`DD_TEST_OPTIMIZATION_RUNNER_VITEST_CONFIG=vitest.ci.config.ts` for both steps. +Paths are relative to the working directory. The option selects a config file; +it does not change the working directory. + +Vitest owns configuration, project discovery, setup/teardown, reporters, and +coverage. DDTest runs once with `run: true` and `watch: false`, and executes only +the discovered project/pool specifications whose canonical file paths belong to +the worker's assignment. The same file can still run in multiple configured +projects. Vitest 3–5 use the public specification API; Vitest 1.6–2 use a separate +legacy adapter. Configuration and discovery errors fail the operation rather +than falling back to a filesystem glob. + +### Migrating from a Vitest command wrapper + +`--command` and `DD_TEST_OPTIMIZATION_RUNNER_COMMAND` are rejected for Vitest, +including direct `vitest`, `node .../vitest.mjs`, `npx`, `pnpm`, and npm-script +invocations. Remove the command override and move Vitest options to its config: + +| Previous command option | Configuration or replacement | +| --- | --- | +| `--config vitest.ci.config.ts` | DDTest's `--vitest-config vitest.ci.config.ts` | +| `--project 'unit*'` | `test.project: ['unit*']` | +| `--testNamePattern smoke` | `test.testNamePattern: 'smoke'` | +| `--reporter json --outputFile results.json` | `test.reporters: ['json']`, `test.outputFile: 'results.json'` | +| `--coverage` | `test.coverage.enabled: true` plus an installed coverage provider | +| `--passWithNoTests` | `test.passWithNoTests: true` | +| Test-file arguments | Positional selections to `ddtest plan`, then `ddtest run` | + +For example, replace `--command 'pnpm exec vitest run --config +vitest.ci.config.ts --project unit*'` with `--vitest-config vitest.ci.config.ts` +and add `project: ['unit*']` under that file's `test` configuration. Regenerate +any saved plan after changing the configuration or selection. + +Shell setup and npm lifecycle scripts are not executed by the adapter. Run +preparation steps before DDTest, and export required environment variables to +DDTest. Keep Node flags and project loaders in `NODE_OPTIONS`; DDTest preserves +them. For Yarn Plug'n'Play, make its loaders available to Node in that environment. +Choose the Node version on `PATH` before starting DDTest. + +Use Vitest directly for watch mode, UI, benchmarks, report merging, and other +CLI-only workflows. For snapshot updates, run Vitest with `--update`, or +explicitly enable `test.update` in a dedicated config. Missing snapshots in CI +still fail by default. Prefer DDTest's worker/CI-node settings for splitting; +a configured Vitest `test.shard` additionally shards each assigned batch. + +### Tracing + +DDTest adds the Datadog CI require and register import to worker `NODE_OPTIONS`. +It preserves an existing Datadog register import; otherwise it uses `DD_TRACE_ESM_IMPORT`, the `register.js` next to an external tracer's `ci` directory, or the project-local `dd-trace/register.js`, in that order. -The GitHub action exports the paths, so manually setting `NODE_OPTIONS` is -unnecessary. -The tracer's `--require` option follows existing project loaders. Discovery -removes both Datadog options to avoid instrumenting the file-listing process. +The GitHub action exports the paths, so manually setting Datadog `NODE_OPTIONS` +is unnecessary. The tracer's require follows existing project loaders. +Discovery removes both Datadog options to avoid instrumenting file listing. +The adapter itself is a directly executed script, not a `NODE_OPTIONS` preload. ## Cypress Discovery And Instrumentation diff --git a/docs/settings.md b/docs/settings.md index bb589490..fb3ef573 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -10,7 +10,8 @@ platforms or frameworks require explicit selection. | --- | --- | --- | ---: | --- | | `--platform` | `DD_TEST_OPTIMIZATION_RUNNER_PLATFORM` | | auto-detected | Language/platform. Currently supported: `ruby`, `python`, `javascript`. | | `--framework` | `DD_TEST_OPTIMIZATION_RUNNER_FRAMEWORK` | | auto-detected | Test framework. Currently supported: `rspec`, `minitest`, `pytest`, `cucumber`, `cypress`, `jest`, `mocha`, `playwright`, `vitest`. | -| `--command` | `DD_TEST_OPTIMIZATION_RUNNER_COMMAND` | | `""` | Override the default base test command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, Playwright, and Vitest run/discovery, and pytest run/discovery (since 1.7.0). DDTest appends selected tests and framework-specific flags. For ddtest versions prior to 1.7.0 with pytest, the command cannot be changed. Pass extra flags with `PYTEST_ADDOPTS`. | +| `--command` | `DD_TEST_OPTIMIZATION_RUNNER_COMMAND` | | `""` | Override the default base test command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, and Playwright run/discovery, and pytest run/discovery (since 1.7.0). DDTest appends selected tests and framework-specific flags. For ddtest versions prior to 1.7.0 with pytest, the command cannot be changed. Pass extra flags with `PYTEST_ADDOPTS`. Vitest rejects this option; use `--vitest-config` and configure options in Vitest. | +| `--vitest-config` | `DD_TEST_OPTIMIZATION_RUNNER_VITEST_CONFIG` | | `""` | Vitest config file used by both planning and execution; defaults to Vitest config discovery. See [migration instructions](running.md#vitest-integration). | | `--min-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM` | | physical CPU count | Minimum count DDTest considers when planning. Interpret it as CI nodes in CI-node mode, or workers in a single-node run. | | `--max-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM` | | physical CPU count | Maximum count DDTest considers when planning. Interpret it as CI nodes in CI-node mode, or workers in a single-node run. | | `--ci-job-overhead` | `DD_TEST_OPTIMIZATION_RUNNER_CI_JOB_OVERHEAD` | | `25s` | Modeled overhead for adding one more CI node. Accepts durations such as `25s`, `1m`, `1500ms`, or `0s` to disable this bias. Increase it to use fewer CI nodes; decrease it to prefer faster wall time. | diff --git a/docs/third-party-runners.md b/docs/third-party-runners.md index 0850315e..bca8e59b 100644 --- a/docs/third-party-runners.md +++ b/docs/third-party-runners.md @@ -47,14 +47,18 @@ fi ## Vitest -When another runner consumes DDTest's file list for Vitest, load both dd-trace -initialization entry points: +Vitest's CLI treats file arguments as substring filters. Passing a DDTest file +list to `vitest run` can execute additional files with overlapping names. +Use [DDTest's Vitest integration](running.md#vitest-integration) to execute exact +assignments, or build the external runner around Vitest's +[specification API](https://vitest.dev/api/advanced/vitest#runtestspecifications) +and retain only specifications whose file paths are in the assignment. + +An external Node API runner must also load both Datadog initialization entry +points and use `dd-trace` 5.125.0 or higher: ```bash export NODE_OPTIONS="--import dd-trace/register.js -r dd-trace/ci/init${NODE_OPTIONS:+ $NODE_OPTIONS}" -if [ -s .testoptimization/runner/test-files.txt ]; then - xargs ./node_modules/.bin/vitest run < .testoptimization/runner/test-files.txt -fi ``` ## Mocha diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 2af5b770..26139a33 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -118,6 +118,7 @@ var rootPersistentFlagBindings = []persistentFlagBinding{ {configKey: "ci_node", flagName: "ci-node"}, {configKey: "ci_node_workers", flagName: "ci-node-workers"}, {configKey: "command", flagName: "command"}, + {configKey: "vitest_config", flagName: "vitest-config"}, {configKey: "tests_location", flagName: "tests-location"}, {configKey: "tests_exclude_pattern", flagName: "tests-exclude-pattern"}, {configKey: "test_discovery_cache", flagName: "test-discovery-cache"}, @@ -139,7 +140,8 @@ func init() { rootCmd.PersistentFlags().String("worker-env", "", "Worker environment configuration") rootCmd.PersistentFlags().Int("ci-node", -1, "CI node index to run (0-indexed; default: -1 disables CI-node mode)") rootCmd.PersistentFlags().String("ci-node-workers", "1", `Number of parallel workers per CI node (positive integer or "ncpu"; default: 1)`) - rootCmd.PersistentFlags().String("command", "", "Test command that ddtest should wrap") + rootCmd.PersistentFlags().String("command", "", "Test command that ddtest should wrap (not supported for Vitest)") + rootCmd.PersistentFlags().String("vitest-config", "", "Vitest configuration file for discovery and execution (defaults to Vitest config discovery)") rootCmd.PersistentFlags().String("tests-location", "", "Glob pattern used to discover test files") rootCmd.PersistentFlags().String("tests-exclude-pattern", "", "Glob pattern used to exclude test files from discovery") rootCmd.PersistentFlags().String("test-discovery-cache", "", "Path to a restored test discovery cache file to import before planning") diff --git a/internal/cmd/cmd_test.go b/internal/cmd/cmd_test.go index 3046ae8d..dc25c8cb 100644 --- a/internal/cmd/cmd_test.go +++ b/internal/cmd/cmd_test.go @@ -651,6 +651,13 @@ func TestFlagBinding(t *testing.T) { t.Fatalf("Error setting target-time flag: %v", err) } + if err := rootCmd.PersistentFlags().Set("vitest-config", "config/vitest.ts"); err != nil { + t.Fatal(err) + } + if viper.GetString("vitest_config") != "config/vitest.ts" { + t.Fatalf("vitest-config was not bound: %q", viper.GetString("vitest_config")) + } + // Check that viper picks up the flag values if viper.GetString("platform") != "python" { t.Errorf("expected viper platform to be 'python', got %q", viper.GetString("platform")) diff --git a/internal/compatibility/vitest_test.go b/internal/compatibility/vitest_test.go index 0cc05af9..c657eb48 100644 --- a/internal/compatibility/vitest_test.go +++ b/internal/compatibility/vitest_test.go @@ -18,13 +18,22 @@ import ( "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/settings" ) +func configureVitest(config string) { + configureFramework("", "") + settings.Get().VitestConfig = config +} + func TestVitestAdapterIntegration(t *testing.T) { nodeModules := requireEnv(t, "DDTEST_VITEST_NODE_MODULES") resetSettingsAfterTest(t) - root := t.TempDir() + root := filepath.Join(t.TempDir(), "project with spaces") + if err := os.MkdirAll(root, 0755); err != nil { + t.Fatal(err) + } if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { t.Fatal(err) } @@ -36,6 +45,7 @@ export default { include: ['checks/**/*.check.js'], setupFiles: ['./setup.js'], globalSetup: ['./global-setup.js'], + passWithNoTests: process.env.DDTEST_PASS_WITH_NO_TESTS === 'true', }, } @@ -67,7 +77,7 @@ test('must not run', () => { }) `) t.Chdir(root) - configureFramework(shellCommand(filepath.Join(root, "node_modules", ".bin", "vitest"), "--config", "vitest.unit.mjs"), "") + configureVitest("vitest.unit.mjs") vitest := framework.NewVitest() ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) @@ -93,7 +103,9 @@ test('must not run', () => { t.Run("exact file membership with overlapping names", func(t *testing.T) { writeFixture(t, root, "vitest.overlap.mjs", `export default { - test: { dir: 'src', include: ['**/test.js'], setupFiles: ['./setup.js'] }, + test: { dir: 'src', include: ['**/test.js'], setupFiles: ['./setup.js'], + shard: process.env.DDTEST_SHARD || undefined, + }, } `) for _, name := range []string{"endOfYear", "eachWeekendOfYear", "otherEndOfYear"} { @@ -107,22 +119,22 @@ test('runs only in its assigned batch', () => { `) } for _, tc := range []struct { - name, command, selected string + name, selected, shard string }{ - {"shard in run mode", shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.overlap.mjs", "--shard=1/1"), "src/endOfYear/test.js"}, - {"direct", shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.overlap.mjs"), "src/endOfYear/test.js"}, - {"node with separator", shellCommand("node", filepath.Join(root, "node_modules/vitest/vitest.mjs"), "run", "--config", "vitest.overlap.mjs", "--", "old.test.js"), filepath.Join(root, "src/endOfYear/test.js")}, - {"package manager", shellCommand("npx", "--no-install", "vitest", "run", "--config", "vitest.overlap.mjs"), "src/eachWeekendOfYear/test.js"}, + {"shard in run mode", "src/endOfYear/test.js", "1/1"}, + {"relative path", "src/endOfYear/test.js", ""}, + {"absolute path", filepath.Join(root, "src/endOfYear/test.js"), ""}, + {"second assignment", "src/eachWeekendOfYear/test.js", ""}, } { t.Run(tc.name, func(t *testing.T) { events := filepath.Join(root, "executed.txt") if err := os.WriteFile(events, nil, 0600); err != nil { t.Fatal(err) } - configureFramework(tc.command, "") + configureVitest("vitest.overlap.mjs") name := filepath.Base(filepath.Dir(tc.selected)) if err := framework.NewVitest().RunTests(ctx, []string{tc.selected}, map[string]string{ - "DDTEST_VITEST_WORKER": name, "DDTEST_VITEST_EVENTS": events, + "DDTEST_VITEST_WORKER": name, "DDTEST_VITEST_EVENTS": events, "DDTEST_SHARD": tc.shard, }); err != nil { t.Fatalf("exact-file run failed: %v", err) } @@ -137,7 +149,7 @@ test('runs only in its assigned batch', () => { } // An empty batch must not turn into an unfiltered full-suite run. - configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.overlap.mjs"), "") + configureVitest("vitest.overlap.mjs") if err := framework.NewVitest().RunTests(ctx, nil, nil); err != nil { t.Fatal(err) } @@ -155,6 +167,8 @@ const major = Number(require('vitest/package.json').version.split('.')[0]) export default { test: { ...(major < 4 ? { workspace: './workspace.mjs' } : { projects: ['./one/vitest.config.mjs', './two/vitest.config.mjs'] }), + project: process.env.DDTEST_PROJECT_FILTER ? [process.env.DDTEST_PROJECT_FILTER] : [], + testNamePattern: 'selected in each project', reporters: ['json'], outputFile: './results.json', }, @@ -162,8 +176,9 @@ export default { `) writeFixture(t, root, "workspace.mjs", `export default ['./one/vitest.config.mjs', './two/vitest.config.mjs']`) for _, name := range []string{"one", "two"} { - writeFixture(t, root, name+"/vitest.config.mjs", `export default { test: { - name: '`+name+`', root: new URL('..', import.meta.url).pathname, include: ['project-checks/*.test.js'], + writeFixture(t, root, name+"/vitest.config.mjs", `import { fileURLToPath } from 'node:url' +export default { test: { + name: '`+name+`', root: fileURLToPath(new URL('..', import.meta.url)), include: ['project-checks/*.test.js'], env: { DDTEST_PROJECT: '`+name+`' }, } } `) @@ -183,15 +198,12 @@ test('must not run', () => { throw new Error('file assignment lost') }) if err := os.WriteFile(events, nil, 0600); err != nil { t.Fatal(err) } - args := []string{"--config", "multi-project.config.mjs", "--testNamePattern", "selected in each project"} want := []string{"one"} - if project != "" { - args = append(args, "--project", project) - } else { + if project == "" { want = append(want, "two") } - configureFramework(shellCommand(append([]string{filepath.Join(root, "node_modules/.bin/vitest")}, args...)...), "") - if err := framework.NewVitest().RunTests(ctx, []string{"project-checks/selected.test.js"}, map[string]string{"DDTEST_VITEST_EVENTS": events}); err != nil { + configureVitest("multi-project.config.mjs") + if err := framework.NewVitest().RunTests(ctx, []string{"project-checks/selected.test.js"}, map[string]string{"DDTEST_VITEST_EVENTS": events, "DDTEST_PROJECT_FILTER": project}); err != nil { t.Fatal(err) } contents, err := os.ReadFile(events) @@ -228,22 +240,20 @@ test('must not run', () => { throw new Error('file assignment lost') }) }) t.Run("CI snapshots require an explicit update", func(t *testing.T) { - writeFixture(t, root, "snapshot.config.mjs", `export default { test: { include: ['snapshot.test.js'] } }`) + writeFixture(t, root, "snapshot.config.mjs", `export default { test: { include: ['snapshot.test.js'], update: process.env.DDTEST_UPDATE_SNAPSHOTS === 'true' } }`) writeFixture(t, root, "snapshot.test.js", `import { expect, test } from 'vitest' test('snapshot policy', () => { expect({ assigned: true }).toMatchSnapshot() }) `) - command := []string{filepath.Join(root, "node_modules/.bin/vitest"), "--config", "snapshot.config.mjs"} - configureFramework(shellCommand(command...), "") + configureVitest("snapshot.config.mjs") err := framework.NewVitest().RunTests(ctx, []string{"snapshot.test.js"}, map[string]string{"CI": "true"}) if err == nil || !strings.Contains(err.Error(), "exit status 1") { t.Fatalf("expected missing-snapshot failure in CI, got %v", err) } snapshot := filepath.Join(root, "__snapshots__/snapshot.test.js.snap") if _, err := os.Stat(snapshot); !os.IsNotExist(err) { - t.Fatalf("snapshot should not be written without --update, got %v", err) + t.Fatalf("snapshot should not be written without explicit update, got %v", err) } - configureFramework(shellCommand(append(command, "--update")...), "") - if err := framework.NewVitest().RunTests(ctx, []string{"snapshot.test.js"}, map[string]string{"CI": "true"}); err != nil { + if err := framework.NewVitest().RunTests(ctx, []string{"snapshot.test.js"}, map[string]string{"CI": "true", "DDTEST_UPDATE_SNAPSHOTS": "true"}); err != nil { t.Fatalf("explicit snapshot update failed: %v", err) } if _, err := os.Stat(snapshot); err != nil { @@ -261,7 +271,7 @@ test('snapshot policy', () => { expect({ assigned: true }).toMatchSnapshot() }) import { add } from './math.js' test('records coverage', () => { expect(add(1, 2)).toBe(3) }) `) - configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "coverage.config.mjs"), "") + configureVitest("coverage.config.mjs") if err := framework.NewVitest().RunTests(ctx, []string{"coverage.test.js"}, nil); err != nil { t.Fatal(err) } @@ -287,14 +297,26 @@ test('records coverage', () => { expect(add(1, 2)).toBe(3) }) t.Fatalf("coverage did not record executed statements in math.js: %s", report) }) + t.Run("configuration errors fail discovery and execution", func(t *testing.T) { + writeFixture(t, root, "broken.config.mjs", `throw new Error('invalid fixture config')`) + configureVitest("broken.config.mjs") + _, err := framework.NewVitest().DiscoverTestFiles(ctx, discovery.TestFileSet{Pattern: "**/*.test.js"}) + if err == nil || !strings.Contains(err.Error(), "broken.config.mjs") { + t.Fatalf("expected configuration error, got %v", err) + } + err = framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, nil) + if err == nil || !strings.Contains(err.Error(), "exit status 1") { + t.Fatalf("expected configuration failure, got %v", err) + } + }) + t.Run("no matching specification", func(t *testing.T) { - configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.unit.mjs"), "") + configureVitest("vitest.unit.mjs") err := framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, nil) if err == nil || !strings.Contains(err.Error(), "exit status 1") { t.Fatalf("expected no-test failure, got %v", err) } - configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest"), "--config", "vitest.unit.mjs", "--passWithNoTests"), "") - if err := framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, nil); err != nil { + if err := framework.NewVitest().RunTests(ctx, []string{"src/endOfYear/test.js"}, map[string]string{"DDTEST_PASS_WITH_NO_TESTS": "true"}); err != nil { t.Fatalf("passWithNoTests failed: %v", err) } }) @@ -342,7 +364,7 @@ test('ddtest unassigned traced file', () => { throw new Error('unassigned file r defer agent.Close() t.Chdir(root) t.Setenv("NODE_OPTIONS", "") - configureFramework(shellCommand(filepath.Join(root, "node_modules/.bin/vitest")), "") + configureVitest("") vitest := framework.NewVitest() tracerRoot := filepath.Join(tracerModules, "dd-trace") vitest.SetPlatformEnv(map[string]string{ diff --git a/internal/framework/command_override_test.go b/internal/framework/command_override_test.go index 001baa5d..8cb6fa61 100644 --- a/internal/framework/command_override_test.go +++ b/internal/framework/command_override_test.go @@ -226,7 +226,7 @@ func TestFrameworkCommand(t *testing.T) { {NewJest(), "npx", []string{"jest"}}, {NewMocha(), "npx", []string{"mocha"}}, {NewCucumber(), "npx", []string{"cucumber-js"}}, - {NewVitest(), "npx", []string{"vitest", "run"}}, + {NewVitest(), "node", nil}, {NewPlaywright(), "npx", []string{"playwright", "test"}}, {NewCypress(), "npx", []string{"cypress", "run"}}, {NewPytest(), "python", []string{"-m", "pytest"}}, @@ -266,7 +266,7 @@ func TestFrameworkCommandPreservesOverride(t *testing.T) { viper.Reset() viper.Set("command", `npm test -- --runInBand "path with spaces"`) settings.Init() - for _, f := range []Framework{NewJest(), NewMocha(), NewCucumber(), NewVitest(), NewPlaywright(), NewCypress(), NewPytest(), NewRSpec(), NewMinitest()} { + for _, f := range []Framework{NewJest(), NewMocha(), NewCucumber(), NewPlaywright(), NewCypress(), NewPytest(), NewRSpec(), NewMinitest()} { command, args := f.Command() if command != "npm" || !slices.Equal(args, []string{"test", "--", "--runInBand", "path with spaces"}) { t.Fatalf("%s: %s %q", f.Name(), command, args) diff --git a/internal/framework/scripts/vitest.mjs b/internal/framework/scripts/vitest.mjs new file mode 100644 index 00000000..62c6d1a4 --- /dev/null +++ b/internal/framework/scripts/vitest.mjs @@ -0,0 +1,44 @@ +import { readFileSync, realpathSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +const request = JSON.parse(readFileSync(new URL('./request.json', import.meta.url), 'utf8')) +// Resolve the project's Vitest, even though this adapter lives in a temp directory. +const require = createRequire(resolve('package.json')) +const { createVitest } = await import(pathToFileURL(require.resolve('vitest/node')).href) +const major = Number(require('vitest/package.json').version.split('.')[0]) +const runner = major >= 3 + ? await import('./vitest_modern.mjs') + : await import('./vitest_legacy.mjs') + +// Match the test environment before loading the user's configuration. +process.env.TEST = 'true' +process.env.VITEST = 'true' +process.env.NODE_ENV ??= 'test' +const config = { config: request.config, run: true, watch: false } +const context = major >= 5 + ? await createVitest(config) + : await createVitest('test', config) + +try { + // Discovery only loads configuration; execution also initializes reporters + // and coverage. Neither step starts tests before assignment filtering. + if (!request.discover) await runner.initialize(context) + const discovered = await runner.discoverSpecifications(context) + if (request.discover) { + writeFileSync(new URL('./files.json', import.meta.url), JSON.stringify(discovered.map(runner.filePath))) + } else { + const canonicalPath = file => realpathSync(resolve(file)) + const selected = new Set(request.files.map(canonicalPath)) + const specs = discovered.filter(spec => selected.has(canonicalPath(runner.filePath(spec)))) + if (specs.length === 0) { + console.error('No assigned Vitest test files found') + process.exitCode = context.config.passWithNoTests ? 0 : 1 + } else { + await runner.runSpecifications(context, specs) + } + } +} finally { + await context.close() +} diff --git a/internal/framework/scripts/vitest_exact_files.mjs b/internal/framework/scripts/vitest_exact_files.mjs deleted file mode 100644 index 3e44b90b..00000000 --- a/internal/framework/scripts/vitest_exact_files.mjs +++ /dev/null @@ -1,61 +0,0 @@ -import { readFileSync, realpathSync } from 'node:fs' -import { createRequire } from 'node:module' -import { basename, resolve } from 'node:path' -import { pathToFileURL } from 'node:url' - -const entrypoint = process.argv[1] || '' -const filesPath = process.env.DDTEST_VITEST_SELECTED_FILES - -// Package managers inherit NODE_OPTIONS. Wait for the actual Vitest process, -// then clear the request so its test workers do not install the adapter again. -if (filesPath && ['vitest', 'vitest.mjs', 'vitest.js'].includes(basename(entrypoint))) { - delete process.env.DDTEST_VITEST_SELECTED_FILES - const canonicalPath = file => realpathSync(resolve(file)) - const selected = new Set(JSON.parse(readFileSync(filesPath, 'utf8')).map(canonicalPath)) - const require = createRequire(pathToFileURL(realpathSync(entrypoint))) - const { createVitest, parseCLI } = await import(pathToFileURL(require.resolve('vitest/node')).href) - const major = Number(require('vitest/package.json').version.split('.')[0]) - const runner = major >= 3 - ? await import('./vitest_exact_files_modern.mjs') - : await import('./vitest_exact_files_legacy.mjs') - // Use Vitest's own parser to preserve command options. Positional filters - // are replaced by the exact assignment. - const { options } = parseCLI(['vitest', ...process.argv.slice(2)]) - if (options.exclude) { - options.cliExclude = options.exclude - delete options.exclude - } - // Match the CLI's environment before loading the user's configuration. - process.env.TEST = 'true' - process.env.VITEST = 'true' - process.env.NODE_ENV ??= 'test' - - // Own the lifecycle so discovery cannot execute files before we filter them. - // dd-trace 5.125+ instruments createVitest directly; no watch mode is needed. - const config = { ...options, run: true, watch: false } - const context = major >= 5 - ? await createVitest(config) - : await createVitest('test', config) - try { - // Initialize reporters and coverage without starting a test run. - await runner.initialize(context) - const discovered = await runner.discoverSpecifications(context) - const specs = discovered.filter(spec => selected.has(canonicalPath(runner.filePath(spec)))) - if (specs.length === 0) { - console.error('No assigned Vitest test files found') - process.exitCode = context.config.passWithNoTests ? 0 : 1 - } else { - await runner.runSpecifications(context, specs) - } - } catch (error) { - console.error(error) - process.exitCode = 1 - } finally { - await context.close() - } - // The API run replaces the CLI entrypoint. Flush reporters before exiting so - // the original CLI cannot start a second run with substring filters. - await Promise.all([process.stdout, process.stderr].map(stream => - new Promise(resolve => stream.write('', resolve)))) - process.exit() -} diff --git a/internal/framework/scripts/vitest_exact_files_legacy.mjs b/internal/framework/scripts/vitest_legacy.mjs similarity index 100% rename from internal/framework/scripts/vitest_exact_files_legacy.mjs rename to internal/framework/scripts/vitest_legacy.mjs diff --git a/internal/framework/scripts/vitest_exact_files_modern.mjs b/internal/framework/scripts/vitest_modern.mjs similarity index 100% rename from internal/framework/scripts/vitest_exact_files_modern.mjs rename to internal/framework/scripts/vitest_modern.mjs diff --git a/internal/framework/scripts/vitest_v1_discovery.mjs b/internal/framework/scripts/vitest_v1_discovery.mjs deleted file mode 100644 index 3a5bdf0a..00000000 --- a/internal/framework/scripts/vitest_v1_discovery.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import { createVitest, parseCLI } from 'vitest/node' - -const outputMarker = '__DDTEST_VITEST_FILES__' -const cliArgs = JSON.parse(process.argv[1]) -const { filter, options } = parseCLI(['vitest', ...cliArgs]) -const vitest = await createVitest('test', { ...options, watch: false }) - -try { - const specs = await vitest.globTestFiles(filter) - const files = specs.map(spec => Array.isArray(spec) ? spec[1] : spec) - process.stdout.write(`${outputMarker}${JSON.stringify(files)}\n`) -} finally { - await vitest.close() -} diff --git a/internal/framework/vitest.go b/internal/framework/vitest.go index 31783567..08399341 100644 --- a/internal/framework/vitest.go +++ b/internal/framework/vitest.go @@ -4,15 +4,12 @@ import ( "context" _ "embed" "encoding/json" - "errors" "fmt" "log/slog" "maps" - "net/url" "os" "path/filepath" "slices" - "strconv" "strings" "github.com/DataDog/ddtest/internal/discovery" @@ -22,43 +19,32 @@ import ( "github.com/DataDog/ddtest/internal/utils" ) -const ( - binVitestPath = "node_modules/.bin/vitest" - ddTraceRegisterPath = "dd-trace/register.js" - vitestV1DiscoveryMarker = "__DDTEST_VITEST_FILES__" - vitestSelectedFilesEnv = "DDTEST_VITEST_SELECTED_FILES" -) - -//go:embed scripts/vitest_v1_discovery.mjs -var vitestV1DiscoveryScript string +const ddTraceRegisterPath = "dd-trace/register.js" -//go:embed scripts/vitest_exact_files.mjs -var vitestExactFilesScript string +//go:embed scripts/vitest.mjs +var vitestScript string -//go:embed scripts/vitest_exact_files_modern.mjs -var vitestExactFilesModernScript string +//go:embed scripts/vitest_modern.mjs +var vitestModernScript string -//go:embed scripts/vitest_exact_files_legacy.mjs -var vitestExactFilesLegacyScript string +//go:embed scripts/vitest_legacy.mjs +var vitestLegacyScript string var vitestTestFileExtensions = []string{"js", "jsx", "ts", "tsx", "mjs", "mts", "cjs", "cts"} -type vitestExecutor interface { - ext.CommandExecutor - Output(ctx context.Context, name string, args []string, envMap map[string]string) ([]byte, []byte, error) -} - type Vitest struct { - executor vitestExecutor - commandOverride []string - platformEnv map[string]string + executor ext.CommandExecutor + configFile string + customCommand string + platformEnv map[string]string } func NewVitest() *Vitest { return &Vitest{ - executor: &ext.DefaultCommandExecutor{}, - commandOverride: loadCommandOverride(), - platformEnv: make(map[string]string), + executor: &ext.DefaultCommandExecutor{}, + configFile: settings.GetVitestConfig(), + customCommand: settings.GetCommand(), + platformEnv: make(map[string]string), } } @@ -103,11 +89,14 @@ func (v *Vitest) DiscoverTests(ctx context.Context, testFiles discovery.TestFile return nil, ErrFullTestDiscoveryUnsupported } -// DiscoverTestFiles uses Vitest's config-aware file listing when available, -// then falls back to the Vitest 1.6 API and finally DDTest's filesystem glob. +// DiscoverTestFiles uses the same config-aware Node adapter as execution. +// A discovery failure is returned to the caller, never replaced with a broader glob. func (v *Vitest) DiscoverTestFiles(ctx context.Context, testFiles discovery.TestFileSet) ([]string, error) { - // With an exclude pattern, ExplicitFiles contains candidates from DDTest's generic - // glob. Vitest discovery must remain authoritative before applying the exclude. + if err := v.validateCommand(); err != nil { + return nil, err + } + // With an exclude pattern, explicit files are generic glob candidates; + // Vitest's configuration must remain authoritative before excluding files. if settings.GetTestsExcludePattern() == "" { if testFiles.Empty() { return []string{}, nil @@ -117,196 +106,95 @@ func (v *Vitest) DiscoverTestFiles(ctx context.Context, testFiles discovery.Test } } - command, baseArgs := v.Command() - outputDir, err := os.MkdirTemp("", "ddtest-vitest-list-*") + dir, err := prepareVitestAdapter(vitestRequest{Config: v.configFile, Discover: true}) if err != nil { - return nil, fmt.Errorf("failed to create Vitest discovery output directory: %w", err) + return nil, err } - defer func() { _ = os.RemoveAll(outputDir) }() - outputFile := filepath.Join(outputDir, "files.json") - - args := vitestArgsForSubcommand(baseArgs, "list") - args = withFrameworkOptions(command, args, "vitest", "--filesOnly", "--json="+outputFile) - - slog.Info("Discovering Vitest test files with command", "command", command, "args", args) - stdout, stderr, err := v.executor.Output(ctx, command, args, v.discoveryEnv()) + defer func() { _ = os.RemoveAll(dir) }() + output, err := v.executor.CombinedOutput(ctx, "node", []string{filepath.Join(dir, "vitest.mjs")}, v.discoveryEnv()) if err != nil { - output := append(slices.Clone(stdout), stderr...) - if supportsVitestV1DiscoveryFallback(output) { - return v.discoverVitestV1TestFiles(ctx, command, baseArgs, testFiles) - } - message := strings.TrimSpace(string(output)) - if message == "" { - return nil, fmt.Errorf("failed to discover Vitest test files: %w", err) - } - return nil, fmt.Errorf("failed to discover Vitest test files: %s: %w", message, err) + return nil, fmt.Errorf("failed to discover Vitest test files: %s: %w", strings.TrimSpace(string(output)), err) } - - if message := strings.TrimSpace(string(stderr)); message != "" { - slog.Debug("Vitest test file discovery wrote to stderr", "output", message) - } - if message := strings.TrimSpace(string(stdout)); message != "" { - slog.Debug("Vitest test file discovery wrote to stdout", "output", message) + if message := strings.TrimSpace(string(output)); message != "" { + slog.Debug("Vitest discovery output", "output", message) } - output, err := os.ReadFile(outputFile) + // Keep the result separate from configuration/plugin logs on stdout. + contents, err := os.ReadFile(filepath.Join(dir, "files.json")) if err != nil { return nil, fmt.Errorf("failed to read Vitest test file list: %w", err) } - discoveredFiles, err := parseVitestListFilesOutput(output) - if err != nil { - message := strings.TrimSpace(string(stderr)) - if message == "" { - return nil, fmt.Errorf("failed to parse Vitest test file list: %w", err) - } - return nil, fmt.Errorf("failed to parse Vitest test file list: %w; stderr: %s", err, message) + var files []string + if err := json.Unmarshal(contents, &files); err != nil { + return nil, fmt.Errorf("failed to parse Vitest test file list: %w", err) } + files = normalizeJavaScriptTestFiles(files) if settings.GetTestsLocation() == "" && settings.GetTestsExcludePattern() == "" { - return discoveredFiles, nil + return files, nil } - - return filterJavaScriptTestFiles(discoveredFiles, testFiles) -} - -// discoverVitestV1TestFiles uses the project's vitest/node API to load its -// configuration and list test files without executing them. -func (v *Vitest) discoverVitestV1TestFiles(ctx context.Context, command string, baseArgs []string, testFiles discovery.TestFileSet) ([]string, error) { - cliArgs, err := json.Marshal(vitestCLIArgs(command, baseArgs)) - if err == nil { - slog.Info("Vitest does not support list --filesOnly; using the Vitest 1.6 config-aware discovery API") - output, discoveryErr := v.executor.CombinedOutput(ctx, "node", []string{ - "--input-type=module", - "--eval", - vitestV1DiscoveryScript, - string(cliArgs), - }, v.discoveryEnv()) - if discoveryErr == nil { - discoveredFiles, parseErr := parseVitestV1DiscoveryOutput(output) - if parseErr == nil { - if settings.GetTestsLocation() == "" && settings.GetTestsExcludePattern() == "" { - return discoveredFiles, nil - } - return filterJavaScriptTestFiles(discoveredFiles, testFiles) - } - err = parseErr - } else { - message := strings.TrimSpace(string(output)) - if message == "" { - err = discoveryErr - } else { - err = fmt.Errorf("%s: %w", message, discoveryErr) - } - } - } - - slog.Warn("Vitest 1.6 config-aware discovery failed; using ddtest glob discovery", "error", err) - return discovery.DiscoverTestFiles(testFiles.Pattern, settings.GetTestsExcludePattern()) + return filterJavaScriptTestFiles(files, testFiles) } func (v *Vitest) RunTests(ctx context.Context, testFiles []string, envMap map[string]string) error { + if err := v.validateCommand(); err != nil { + return err + } if len(testFiles) == 0 { return nil } - command, baseArgs := v.Command() - args := vitestArgsForSubcommand(baseArgs, "run") - args = withFrameworkFiles(command, args, "vitest", testFiles) - - slog.Info("Running tests with command", "command", command, "args", args) - - mergedEnv := make(map[string]string) - maps.Copy(mergedEnv, v.platformEnv) - maps.Copy(mergedEnv, envMap) - adapterDir, adapterEnv, err := prepareVitestExactFiles(mergedEnv, testFiles) + dir, err := prepareVitestAdapter(vitestRequest{Config: v.configFile, Files: testFiles}) if err != nil { return err } - defer func() { _ = os.RemoveAll(adapterDir) }() - return v.executor.Run(ctx, command, args, adapterEnv) -} - -// prepareVitestExactFiles writes a worker's adapter and assignment, then builds -// the environment that loads them. The caller must remove adapterDir after the run. -func prepareVitestExactFiles(baseEnv map[string]string, testFiles []string) (string, map[string]string, error) { - assignedFilesJSON, err := encodeVitestAssignedFiles(testFiles) - if err != nil { - return "", nil, err - } - - adapterDir, err := os.MkdirTemp("", "ddtest-vitest-run-*") - if err != nil { - return "", nil, fmt.Errorf("failed to create Vitest execution adapter: %w", err) - } - if err := writeVitestExecutionAdapter(adapterDir, assignedFilesJSON); err != nil { - _ = os.RemoveAll(adapterDir) - return "", nil, err - } + defer func() { _ = os.RemoveAll(dir) }() - return adapterDir, vitestExactFilesEnv(baseEnv, adapterDir), nil + env := make(map[string]string) + maps.Copy(env, v.platformEnv) + maps.Copy(env, envMap) + slog.Info("Running assigned Vitest files with Node API", "config", v.configFile, "files", testFiles) + return v.executor.Run(ctx, "node", []string{filepath.Join(dir, "vitest.mjs")}, env) } -func encodeVitestAssignedFiles(testFiles []string) ([]byte, error) { - // Resolve paths in ddtest's working directory before any command wrapper starts. - // The Node adapter resolves symlinks when comparing these paths with Vitest's specs. - absoluteFiles := make([]string, len(testFiles)) - for i, file := range testFiles { - absolute, err := filepath.Abs(file) - if err != nil { - return nil, fmt.Errorf("failed to resolve Vitest test file: %w", err) - } - absoluteFiles[i] = absolute - } - encodedFiles, err := json.Marshal(absoluteFiles) - if err != nil { - return nil, fmt.Errorf("failed to encode Vitest test files: %w", err) +func (v *Vitest) validateCommand() error { + if strings.TrimSpace(v.customCommand) != "" { + return fmt.Errorf("Vitest uses DDTest's Node API adapter and does not support --command; move options to vitest.config.ts or select a config with --vitest-config (see docs/running.md#vitest-integration)") } - return encodedFiles, nil + return nil } -func writeVitestExecutionAdapter(adapterDir string, assignedFilesJSON []byte) error { - // Keep the modules together so the entrypoint's relative imports work. - // JSON keeps the assignment independent of command-line quoting. - files := []struct { - name string - contents []byte - }{ - {"exact-files.mjs", []byte(vitestExactFilesScript)}, - {"vitest_exact_files_modern.mjs", []byte(vitestExactFilesModernScript)}, - {"vitest_exact_files_legacy.mjs", []byte(vitestExactFilesLegacyScript)}, - {"files.json", assignedFilesJSON}, - } - for _, file := range files { - path := filepath.Join(adapterDir, file.name) - if err := os.WriteFile(path, file.contents, 0600); err != nil { - return fmt.Errorf("failed to write Vitest execution adapter file %q: %w", file.name, err) - } - } - return nil +// Command identifies the adapter's runtime. RunTests supplies its temporary script. +func (v *Vitest) Command() (string, []string) { + return "node", nil } -func vitestExactFilesEnv(baseEnv map[string]string, adapterDir string) map[string]string { - adapterEnv := make(map[string]string, len(baseEnv)+2) - maps.Copy(adapterEnv, baseEnv) +type vitestRequest struct { + Config string `json:"config,omitempty"` + Discover bool `json:"discover,omitempty"` + Files []string `json:"files,omitempty"` +} - // Preserve worker options, including tracing hooks. Inherit the process options - // only when the worker hasn't set NODE_OPTIONS (an explicit empty value wins). - nodeOptions, ok := baseEnv[nodeOptionsEnvVar] - if !ok { - nodeOptions = os.Getenv(nodeOptionsEnvVar) +// prepareVitestAdapter keeps the entrypoint, version adapters, and request +// together. The caller owns the directory and removes it after Node exits. +func prepareVitestAdapter(request vitestRequest) (string, error) { + encoded, err := json.Marshal(request) + if err != nil { + return "", fmt.Errorf("failed to encode Vitest request: %w", err) } - - // NODE_OPTIONS also reaches Vitest launched through npx/pnpm. The entrypoint - // activates only in the Vitest process and reads its assignment from files.json. - adapterPath := filepath.Join(adapterDir, "exact-files.mjs") - // File URLs need forward slashes and a leading slash, including on Windows. - importPath := filepath.ToSlash(adapterPath) - if !strings.HasPrefix(importPath, "/") { - importPath = "/" + importPath + dir, err := os.MkdirTemp("", "ddtest-vitest-*") + if err != nil { + return "", fmt.Errorf("failed to create Vitest adapter directory: %w", err) + } + for name, contents := range map[string]string{ + "vitest.mjs": vitestScript, + "vitest_modern.mjs": vitestModernScript, + "vitest_legacy.mjs": vitestLegacyScript, + "request.json": string(encoded), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + _ = os.RemoveAll(dir) + return "", fmt.Errorf("failed to write Vitest adapter file %q: %w", name, err) + } } - importURL := url.URL{Scheme: "file", Path: importPath} - // Quote the URL so NODE_OPTIONS also works when the temp path contains spaces. - importOption := "--import " + strconv.Quote(importURL.String()) - adapterEnv[nodeOptionsEnvVar] = strings.TrimSpace(nodeOptions + " " + importOption) - adapterEnv[vitestSelectedFilesEnv] = filepath.Join(adapterDir, "files.json") - return adapterEnv + return dir, nil } func (v *Vitest) discoveryEnv() map[string]string { @@ -327,99 +215,6 @@ func (v *Vitest) discoveryEnv() map[string]string { return envMap } -// Decide between a user custom command, the local Vitest binary and npx. -func (v *Vitest) Command() (string, []string) { - if len(v.commandOverride) > 0 { - return v.commandOverride[0], v.commandOverride[1:] - } - - if info, err := os.Stat(binVitestPath); err == nil && !info.IsDir() && info.Mode()&0111 != 0 { - slog.Debug("Using local Vitest binary") - return binVitestPath, []string{"run"} - } - - slog.Debug("Using npx vitest for Vitest commands") - return "npx", []string{"vitest", "run"} -} - -func vitestArgsForSubcommand(baseArgs []string, subcommand string) []string { - args := slices.Clone(baseArgs) - subcommandIndex := 0 - for i, arg := range args { - if isVitestExecutable(arg) { - subcommandIndex = i + 1 - break - } - } - - if subcommandIndex < len(args) { - switch args[subcommandIndex] { - case "run", "watch", "dev", "list": - args[subcommandIndex] = subcommand - return args - } - } - - return slices.Insert(args, subcommandIndex, subcommand) -} - -func vitestCLIArgs(command string, baseArgs []string) []string { - if isVitestExecutable(command) { - return append([]string{}, baseArgs...) - } - - for i, arg := range baseArgs { - if isVitestExecutable(arg) { - return append([]string{}, baseArgs[i+1:]...) - } - } - - return []string{} -} - -func isVitestExecutable(value string) bool { - base := filepath.Base(value) - return base == "vitest" || base == "vitest.mjs" -} - -func supportsVitestV1DiscoveryFallback(output []byte) bool { - message := strings.ToLower(string(output)) - return strings.Contains(message, "unknown option") && - (strings.Contains(message, "filesonly") || strings.Contains(message, "files-only")) -} - -func parseVitestListFilesOutput(output []byte) ([]string, error) { - var listedFiles []struct { - File string `json:"file"` - } - if err := json.Unmarshal(output, &listedFiles); err != nil { - return nil, err - } - - paths := make([]string, 0, len(listedFiles)) - for _, listedFile := range listedFiles { - paths = append(paths, listedFile.File) - } - return normalizeJavaScriptTestFiles(paths), nil -} - -func parseVitestV1DiscoveryOutput(output []byte) ([]string, error) { - markerIndex := strings.LastIndex(string(output), vitestV1DiscoveryMarker) - if markerIndex < 0 { - return nil, errors.New("Vitest 1.6 discovery output did not contain a file list") - } - - encodedFiles := string(output[markerIndex+len(vitestV1DiscoveryMarker):]) - if lineEnd := strings.IndexByte(encodedFiles, '\n'); lineEnd >= 0 { - encodedFiles = encodedFiles[:lineEnd] - } - var testFiles []string - if err := json.Unmarshal([]byte(encodedFiles), &testFiles); err != nil { - return nil, fmt.Errorf("failed to parse Vitest 1.6 discovery output: %w", err) - } - return normalizeJavaScriptTestFiles(testFiles), nil -} - func stripNodeOptionsImport(nodeOptions string, module string) string { return utils.NodeOptionsWithoutImport(nodeOptions, module) } diff --git a/internal/framework/vitest_test.go b/internal/framework/vitest_test.go index 74b3afbc..4d3abb70 100644 --- a/internal/framework/vitest_test.go +++ b/internal/framework/vitest_test.go @@ -13,89 +13,54 @@ import ( "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/settings" + "github.com/spf13/viper" ) type vitestCommandExecutor struct { - onRun func(string, []string, map[string]string) error - output []byte - stdout []byte - stderr []byte - err error - capturedName string - capturedArgs []string - capturedEnvMap map[string]string + onRun func(string, []string, map[string]string) error + output, stdout, stderr []byte + err error + capturedName string + capturedArgs []string + capturedEnvMap map[string]string + capturedRequest vitestRequest } -type vitestCommandCall struct { - name string - args []string -} - -type vitestListOutputEntry struct { - File string `json:"file"` - ProjectName string `json:"projectName,omitempty"` -} - -func vitestListOutput(t *testing.T, entries ...vitestListOutputEntry) []byte { +func vitestListOutput(t *testing.T, files ...string) []byte { t.Helper() - output, err := json.Marshal(entries) + output, err := json.Marshal(files) if err != nil { t.Fatal(err) } return output } -type vitestSequenceExecutor struct { - outputs [][]byte - errors []error - calls []vitestCommandCall -} - -func (m *vitestSequenceExecutor) CombinedOutput(_ context.Context, name string, args []string, _ map[string]string) ([]byte, error) { - callIndex := len(m.calls) - m.calls = append(m.calls, vitestCommandCall{name: name, args: slices.Clone(args)}) - return m.outputs[callIndex], m.errors[callIndex] -} - -func (m *vitestSequenceExecutor) Output(ctx context.Context, name string, args []string, envMap map[string]string) ([]byte, []byte, error) { - output, err := m.CombinedOutput(ctx, name, args, envMap) - return output, nil, err -} - -func (m *vitestSequenceExecutor) Run(_ context.Context, _ string, _ []string, _ map[string]string) error { - return nil -} - -func (m *vitestCommandExecutor) CombinedOutput(_ context.Context, name string, args []string, envMap map[string]string) ([]byte, error) { - m.capturedName = name - m.capturedArgs = slices.Clone(args) - m.capturedEnvMap = envMap - return append(slices.Clone(m.output), m.stderr...), m.err +func (m *vitestCommandExecutor) capture(name string, args []string, env map[string]string) error { + m.capturedName, m.capturedArgs, m.capturedEnvMap = name, slices.Clone(args), env + request, err := os.ReadFile(filepath.Join(filepath.Dir(args[0]), "request.json")) + if err != nil { + return err + } + return json.Unmarshal(request, &m.capturedRequest) } -func (m *vitestCommandExecutor) Output(_ context.Context, name string, args []string, envMap map[string]string) ([]byte, []byte, error) { - m.capturedName = name - m.capturedArgs = slices.Clone(args) - m.capturedEnvMap = envMap - if m.err == nil { - for _, arg := range args { - if outputFile, ok := strings.CutPrefix(arg, "--json="); ok { - if err := os.WriteFile(outputFile, m.output, 0644); err != nil { - return m.stdout, m.stderr, err - } - return m.stdout, m.stderr, nil - } - } +func (m *vitestCommandExecutor) CombinedOutput(_ context.Context, name string, args []string, env map[string]string) ([]byte, error) { + if err := m.capture(name, args, env); err != nil { + return nil, err + } + if m.err != nil { + return m.output, m.err } - return m.output, m.stderr, m.err + err := os.WriteFile(filepath.Join(filepath.Dir(args[0]), "files.json"), m.output, 0600) + return append(slices.Clone(m.stdout), m.stderr...), err } -func (m *vitestCommandExecutor) Run(_ context.Context, name string, args []string, envMap map[string]string) error { - m.capturedName = name - m.capturedArgs = slices.Clone(args) - m.capturedEnvMap = envMap +func (m *vitestCommandExecutor) Run(_ context.Context, name string, args []string, env map[string]string) error { + if err := m.capture(name, args, env); err != nil { + return err + } if m.onRun != nil { - return m.onRun(name, args, envMap) + return m.onRun(name, args, env) } return m.err } @@ -137,7 +102,7 @@ func TestVitest_HasUnskippableMarker(t *testing.T) { } } -func TestVitest_DiscoverTestFiles_WithCustomCommand(t *testing.T) { +func TestVitest_DiscoverTestFiles_WithConfig(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -155,14 +120,14 @@ func TestVitest_DiscoverTestFiles_WithCustomCommand(t *testing.T) { executor := &vitestCommandExecutor{ output: vitestListOutput(t, - vitestListOutputEntry{File: filepath.Join(tempDir, "packages", "b.spec.ts"), ProjectName: "integration"}, - vitestListOutputEntry{File: "packages/a.test.ts", ProjectName: "unit"}, - vitestListOutputEntry{File: "packages/a.test.ts", ProjectName: "duplicate"}, + filepath.Join(tempDir, "packages", "b.spec.ts"), + "packages/a.test.ts", + "packages/a.test.ts", ), } vitest := &Vitest{ - executor: executor, - commandOverride: []string{"pnpm", "exec", "vitest", "run", "--project", "unit*"}, + executor: executor, + configFile: "vitest.unit.ts", platformEnv: map[string]string{ "NODE_OPTIONS": "--import dd-trace/register.js -r dd-trace/ci/init --max-old-space-size=4096", }, @@ -172,14 +137,14 @@ func TestVitest_DiscoverTestFiles_WithCustomCommand(t *testing.T) { if err != nil { t.Fatalf("DiscoverTestFiles() failed: %v", err) } - if executor.capturedName != "pnpm" { - t.Fatalf("command = %q, want pnpm", executor.capturedName) + if executor.capturedName != "node" || len(executor.capturedArgs) != 1 || filepath.Base(executor.capturedArgs[0]) != "vitest.mjs" { + t.Fatalf("command = %q %q", executor.capturedName, executor.capturedArgs) } - wantArgs := []string{"exec", "vitest", "list", "--project", "unit*", "--filesOnly"} - if len(executor.capturedArgs) != len(wantArgs)+1 || - !slices.Equal(executor.capturedArgs[:len(wantArgs)], wantArgs) || - !strings.HasPrefix(executor.capturedArgs[len(wantArgs)], "--json=") { - t.Fatalf("args = %v, want %v followed by --json=", executor.capturedArgs, wantArgs) + if executor.capturedRequest.Config != "vitest.unit.ts" || !executor.capturedRequest.Discover { + t.Fatalf("request = %+v", executor.capturedRequest) + } + if _, err := os.Stat(filepath.Dir(executor.capturedArgs[0])); !os.IsNotExist(err) { + t.Fatalf("discovery adapter directory was not cleaned up: %v", err) } if got := executor.capturedEnvMap["NODE_OPTIONS"]; got != "--max-old-space-size=4096" { t.Fatalf("discovery NODE_OPTIONS = %q", got) @@ -220,9 +185,9 @@ func TestVitest_DiscoverTestFiles_ExcludeStillUsesVitestDiscovery(t *testing.T) vitest := &Vitest{ executor: &vitestCommandExecutor{ output: vitestListOutput(t, - vitestListOutputEntry{File: "generic.test.ts", ProjectName: "unit"}, - vitestListOutputEntry{File: "excluded.test.ts", ProjectName: "unit"}, - vitestListOutputEntry{File: "custom.check.ts", ProjectName: "unit"}, + "generic.test.ts", + "excluded.test.ts", + "custom.check.ts", ), }, platformEnv: make(map[string]string), @@ -262,8 +227,8 @@ func TestVitest_DiscoverTestFiles_ExcludeWithEmptyCandidatesStillUsesVitestDisco setTestsExcludePattern(t, "excluded.test.ts") executor := &vitestCommandExecutor{output: vitestListOutput(t, - vitestListOutputEntry{File: "excluded.test.ts", ProjectName: "unit"}, - vitestListOutputEntry{File: "custom.check.ts", ProjectName: "unit"}, + "excluded.test.ts", + "custom.check.ts", )} vitest := &Vitest{executor: executor, platformEnv: make(map[string]string)} resolvedTestFiles, err := discovery.ResolveTestFiles(vitest.TestPattern(), settings.GetTestsExcludePattern()) @@ -316,7 +281,7 @@ func TestVitest_DiscoverTestFiles_IgnoresStdoutAndStderrNoise(t *testing.T) { t.Fatal(err) } executor := &vitestCommandExecutor{ - output: vitestListOutput(t, vitestListOutputEntry{File: testFile, ProjectName: "unit"}), + output: vitestListOutput(t, testFile), stdout: []byte("Vitest config log\n"), stderr: []byte("Vite deprecation warning\n"), } @@ -330,86 +295,6 @@ func TestVitest_DiscoverTestFiles_IgnoresStdoutAndStderrNoise(t *testing.T) { } } -func TestVitest_DiscoverTestFiles_Vitest16UsesConfigAwareFallback(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - if err := os.Chdir(tempDir); err != nil { - t.Fatal(err) - } - for _, file := range []string{"packages/a.test.ts", "custom/b.check.ts"} { - if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("test"), 0644); err != nil { - t.Fatal(err) - } - } - - executor := &vitestSequenceExecutor{ - outputs: [][]byte{ - []byte("CACError: Unknown option `--filesOnly`"), - []byte("config log\n" + vitestV1DiscoveryMarker + `["packages/a.test.ts","custom/b.check.ts"]` + "\nclose log\n"), - }, - errors: []error{errors.New("exit status 1"), nil}, - } - vitest := &Vitest{ - executor: executor, - commandOverride: []string{"pnpm", "exec", "vitest", "run", "--project", "unit*"}, - platformEnv: make(map[string]string), - } - - files, err := vitest.DiscoverTestFiles(context.Background(), discovery.TestFileSet{Pattern: vitest.TestPattern()}) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(files, []string{"custom/b.check.ts", "packages/a.test.ts"}) { - t.Fatalf("files = %v", files) - } - if len(executor.calls) != 2 { - t.Fatalf("calls = %v", executor.calls) - } - nativeArgs := executor.calls[0].args - wantNativeArgs := []string{"exec", "vitest", "list", "--project", "unit*", "--filesOnly"} - if executor.calls[0].name != "pnpm" || - len(nativeArgs) != len(wantNativeArgs)+1 || - !slices.Equal(nativeArgs[:len(wantNativeArgs)], wantNativeArgs) || - !strings.HasPrefix(nativeArgs[len(wantNativeArgs)], "--json=") { - t.Fatalf("native discovery call = %#v", executor.calls[0]) - } - if executor.calls[1].name != "node" || len(executor.calls[1].args) != 4 { - t.Fatalf("Vitest 1.6 discovery call = %#v", executor.calls[1]) - } - if got := executor.calls[1].args[3]; got != `["run","--project","unit*"]` { - t.Fatalf("Vitest 1.6 CLI args = %s", got) - } -} - -func TestVitest_DiscoverTestFiles_Vitest16FallsBackToDDTestGlob(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - if err := os.Chdir(tempDir); err != nil { - t.Fatal(err) - } - if err := os.WriteFile("fallback.test.ts", []byte("test"), 0644); err != nil { - t.Fatal(err) - } - - executor := &vitestSequenceExecutor{ - outputs: [][]byte{[]byte("Unknown option --filesOnly"), []byte("failed to import vitest/node")}, - errors: []error{errors.New("exit status 1"), errors.New("exit status 1")}, - } - vitest := &Vitest{executor: executor, platformEnv: make(map[string]string)} - files, err := vitest.DiscoverTestFiles(context.Background(), discovery.TestFileSet{Pattern: vitest.TestPattern()}) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(files, []string{"fallback.test.ts"}) { - t.Fatalf("files = %v", files) - } -} - func TestVitest_DiscoverTestFiles_FiltersCustomLocation(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() @@ -428,8 +313,8 @@ func TestVitest_DiscoverTestFiles_FiltersCustomLocation(t *testing.T) { setTestsLocation(t, "custom/**/*.check.ts") executor := &vitestCommandExecutor{output: vitestListOutput(t, - vitestListOutputEntry{File: "src/b.test.ts", ProjectName: "unit"}, - vitestListOutputEntry{File: "custom/a.check.ts", ProjectName: "unit"}, + "src/b.test.ts", + "custom/a.check.ts", )} vitest := &Vitest{executor: executor, platformEnv: make(map[string]string)} files, err := vitest.DiscoverTestFiles(context.Background(), discovery.TestFileSet{Pattern: vitest.TestPattern()}) @@ -441,82 +326,70 @@ func TestVitest_DiscoverTestFiles_FiltersCustomLocation(t *testing.T) { } } -func TestVitest_RunTests_WithCustomCommand(t *testing.T) { +func TestVitest_RejectsCustomCommand(t *testing.T) { executor := &vitestCommandExecutor{} - vitest := &Vitest{ - executor: executor, - commandOverride: []string{"pnpm", "exec", "vitest", "list", "--project", "unit*"}, - platformEnv: map[string]string{"NODE_OPTIONS": "--import dd-trace/register.js -r dd-trace/ci/init", "SHARED": "platform"}, - } - err := vitest.RunTests(context.Background(), []string{"src/a.test.ts", "src/b.spec.ts"}, map[string]string{"SHARED": "worker"}) - if err != nil { - t.Fatal(err) - } - wantArgs := []string{"exec", "vitest", "run", "--project", "unit*", "src/a.test.ts", "src/b.spec.ts"} - if executor.capturedName != "pnpm" || !slices.Equal(executor.capturedArgs, wantArgs) { - t.Fatalf("command = %q, args = %v", executor.capturedName, executor.capturedArgs) + vitest := &Vitest{executor: executor, customCommand: "pnpm exec vitest run"} + _, discoveryErr := vitest.DiscoverTestFiles(t.Context(), discovery.TestFileSet{ExplicitFiles: []string{"a.test.js"}}) + runErr := vitest.RunTests(t.Context(), []string{"a.test.js"}, nil) + for _, err := range []error{discoveryErr, runErr} { + if err == nil || !strings.Contains(err.Error(), "does not support --command") || !strings.Contains(err.Error(), "--vitest-config") { + t.Fatalf("expected migration instructions, got %v", err) + } } - if executor.capturedEnvMap["SHARED"] != "worker" { - t.Fatal("worker environment should override platform environment") + if executor.capturedName != "" { + t.Fatal("custom command must not be executed") } } -func TestVitest_RunTests_UsesNpxFallback(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - if err := os.Chdir(tempDir); err != nil { - t.Fatal(err) - } - - executor := &vitestCommandExecutor{} - vitest := &Vitest{executor: executor, platformEnv: make(map[string]string)} - if err := vitest.RunTests(context.Background(), []string{"src/a.test.ts"}, nil); err != nil { - t.Fatal(err) - } - wantArgs := []string{"vitest", "run", "src/a.test.ts"} - if executor.capturedName != "npx" || !slices.Equal(executor.capturedArgs, wantArgs) { - t.Fatalf("command = %q, args = %v", executor.capturedName, executor.capturedArgs) +func TestVitest_ConfigFromEnvironment(t *testing.T) { + t.Cleanup(func() { viper.Reset(); settings.Init() }) + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_VITEST_CONFIG", "config with spaces/vitest.ts") + viper.Reset() + settings.Init() + if got := NewVitest().configFile; got != "config with spaces/vitest.ts" { + t.Fatalf("config = %q", got) } } func TestVitest_RunTests_ExactSelectionEnvironmentAndCleanup(t *testing.T) { for _, runErr := range []error{nil, errors.New("test process failed")} { t.Run(fmt.Sprintf("run error %v", runErr), func(t *testing.T) { - var selectionPath string - workerEnv := map[string]string{"NODE_OPTIONS": "--max-old-space-size=2048"} - executor := &vitestCommandExecutor{onRun: func(_ string, _ []string, env map[string]string) error { - selectionPath = env[vitestSelectedFilesEnv] - contents, err := os.ReadFile(selectionPath) + var adapterDir string + workerEnv := map[string]string{"NODE_OPTIONS": "--max-old-space-size=2048", "SHARED": "worker"} + executor := &vitestCommandExecutor{onRun: func(name string, args []string, env map[string]string) error { + if name != "node" || len(args) != 1 { + t.Fatalf("command = %q %q", name, args) + } + adapterDir = filepath.Dir(args[0]) + contents, err := os.ReadFile(filepath.Join(adapterDir, "request.json")) if err != nil { t.Fatal(err) } - var files []string - if err := json.Unmarshal(contents, &files); err != nil { + var request vitestRequest + if err := json.Unmarshal(contents, &request); err != nil { t.Fatal(err) } - absolute, err := filepath.Abs("src/endOfYear/test.ts") - if err != nil || !slices.Equal(files, []string{absolute}) { - t.Fatalf("selected files = %v, error = %v", files, err) + if request.Discover || request.Config != "vitest.unit.ts" || !slices.Equal(request.Files, []string{"src/endOfYear/test.ts"}) { + t.Fatalf("request = %+v", request) } - if !strings.HasPrefix(env["NODE_OPTIONS"], workerEnv["NODE_OPTIONS"]+" --import ") { - t.Fatalf("lost worker NODE_OPTIONS: %q", env["NODE_OPTIONS"]) + if env["NODE_OPTIONS"] != workerEnv["NODE_OPTIONS"] || env["SHARED"] != "worker" { + t.Fatalf("worker environment was changed: %v", env) } - for _, name := range []string{"exact-files.mjs", "vitest_exact_files_modern.mjs", "vitest_exact_files_legacy.mjs"} { - if _, err := os.Stat(filepath.Join(filepath.Dir(selectionPath), name)); err != nil { + for _, name := range []string{"vitest.mjs", "vitest_modern.mjs", "vitest_legacy.mjs"} { + if _, err := os.Stat(filepath.Join(adapterDir, name)); err != nil { t.Fatal(err) } } return runErr }} - vitest := &Vitest{executor: executor, platformEnv: map[string]string{"NODE_OPTIONS": "platform-options"}} + vitest := &Vitest{executor: executor, configFile: "vitest.unit.ts", platformEnv: map[string]string{"NODE_OPTIONS": "platform-options", "SHARED": "platform"}} if err := vitest.RunTests(t.Context(), []string{"src/endOfYear/test.ts"}, workerEnv); !errors.Is(err, runErr) { t.Fatalf("got %v, want %v", err, runErr) } - if _, err := os.Stat(filepath.Dir(selectionPath)); !os.IsNotExist(err) { + if _, err := os.Stat(adapterDir); !os.IsNotExist(err) { t.Fatalf("adapter directory was not cleaned up: %v", err) } - if len(workerEnv) != 1 || workerEnv["NODE_OPTIONS"] != "--max-old-space-size=2048" { + if len(workerEnv) != 2 || workerEnv["NODE_OPTIONS"] != "--max-old-space-size=2048" { t.Fatalf("mutated worker environment: %v", workerEnv) } }) @@ -533,49 +406,6 @@ func TestVitest_RunTests_EmptyBatch(t *testing.T) { } } -func TestVitestArgsForSubcommand(t *testing.T) { - tests := []struct { - name string - args []string - want []string - }{ - {name: "replace run", args: []string{"exec", "vitest", "run", "--project", "unit"}, want: []string{"exec", "vitest", "list", "--project", "unit"}}, - {name: "insert after vitest", args: []string{"vitest", "--project", "unit"}, want: []string{"vitest", "list", "--project", "unit"}}, - {name: "project named run", args: []string{"exec", "vitest", "--project", "run"}, want: []string{"exec", "vitest", "list", "--project", "run"}}, - {name: "direct binary args", args: []string{"--project", "unit"}, want: []string{"list", "--project", "unit"}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := vitestArgsForSubcommand(tt.args, "list"); !slices.Equal(got, tt.want) { - t.Fatalf("got %v, want %v", got, tt.want) - } - }) - } -} - -func TestVitestCLIArgs(t *testing.T) { - tests := []struct { - name string - command string - args []string - want []string - }{ - {name: "package manager", command: "pnpm", args: []string{"exec", "vitest", "run", "--project", "unit"}, want: []string{"run", "--project", "unit"}}, - {name: "direct binary without subcommand", command: "node_modules/.bin/vitest", args: []string{"--config", "vitest.unit.ts"}, want: []string{"--config", "vitest.unit.ts"}}, - {name: "npx default", command: "npx", args: []string{"vitest"}, want: nil}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := vitestCLIArgs(tt.command, tt.args); !slices.Equal(got, tt.want) { - t.Fatalf("got %v, want %v", got, tt.want) - } - }) - } - if got := vitestCLIArgs("node_modules/.bin/vitest", nil); got == nil { - t.Fatal("direct binary CLI args must encode as an empty JSON array, not null") - } -} - func TestStripNodeOptionsImport(t *testing.T) { input := "--import dd-trace/register.js --import=other/register.js --max-old-space-size=4096" want := "--import=other/register.js --max-old-space-size=4096" diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 017f9e1a..b7b538b8 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -563,6 +563,7 @@ func TestPrintDDTestSettingsReport_AllSupportedSettings(t *testing.T) { config.Platform = "python" config.Framework = "pytest" config.Command = "pytest -q" + config.VitestConfig = "vitest.ci.config.ts" config.MinParallelism++ config.MaxParallelism += 2 config.ParallelRunnerOverhead += time.Second @@ -608,6 +609,7 @@ func TestPrintDDTestSettingsReport_AllSupportedSettings(t *testing.T) { "CI node", "CI node workers", "Command", + "Vitest config", "Tests location", "Positional selection", "Tests exclude pattern", diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 54809459..5f64f8b9 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -124,6 +124,7 @@ type Config struct { CiNode int `mapstructure:"ci_node"` CiNodeWorkers int `mapstructure:"ci_node_workers"` Command string `mapstructure:"command"` + VitestConfig string `mapstructure:"vitest_config"` TestsLocation string `mapstructure:"tests_location"` TestsSelectionPattern string `mapstructure:"-"` // CLI positional scope, applied after discovery. TestsExcludePattern string `mapstructure:"tests_exclude_pattern"` @@ -228,6 +229,7 @@ func setDefaults() { viper.SetDefault("ci_node", -1) viper.SetDefault("ci_node_workers", strconv.Itoa(defaultCiNodeWorkers)) viper.SetDefault("command", "") + viper.SetDefault("vitest_config", "") viper.SetDefault("tests_location", "") viper.SetDefault("tests_exclude_pattern", "") viper.SetDefault("test_discovery_cache", "") @@ -339,6 +341,10 @@ func GetCommand() string { return Get().Command } +func GetVitestConfig() string { + return Get().VitestConfig +} + func GetTestsLocation() string { return Get().TestsLocation } From f09d52bee77ba85290d99e2262c8c8f010b97132 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Oct 2026 20:26:11 +0200 Subject: [PATCH 08/10] Simplify command flag help text --- internal/cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 26139a33..bbf0cb32 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -140,7 +140,7 @@ func init() { rootCmd.PersistentFlags().String("worker-env", "", "Worker environment configuration") rootCmd.PersistentFlags().Int("ci-node", -1, "CI node index to run (0-indexed; default: -1 disables CI-node mode)") rootCmd.PersistentFlags().String("ci-node-workers", "1", `Number of parallel workers per CI node (positive integer or "ncpu"; default: 1)`) - rootCmd.PersistentFlags().String("command", "", "Test command that ddtest should wrap (not supported for Vitest)") + rootCmd.PersistentFlags().String("command", "", "Test command that ddtest should wrap") rootCmd.PersistentFlags().String("vitest-config", "", "Vitest configuration file for discovery and execution (defaults to Vitest config discovery)") rootCmd.PersistentFlags().String("tests-location", "", "Glob pattern used to discover test files") rootCmd.PersistentFlags().String("tests-exclude-pattern", "", "Glob pattern used to exclude test files from discovery") From 4c24ecb516f9c5fc3580a7383c63d6b0f3f1d165 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 8 Oct 2026 11:43:09 +0200 Subject: [PATCH 09/10] Simplify command option documentation --- README.md | 2 +- docs/settings.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2fa38ca5..1a471026 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ parallelism details, see [Running DDTest](docs/running.md). | --- | --- | | `--platform` | Language/platform. Currently supported: `ruby`, `python`, `javascript`. | | `--framework` | Test framework. Currently supported: `rspec`, `minitest`, `pytest`, `cucumber`, `cypress`, `jest`, `mocha`, `playwright`, `vitest`. | -| `--command` | Override the default base command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, and Playwright run/discovery, and pytest run/discovery (since 1.7.0). For ddtest versions prior to 1.7.0 with pytest, the command cannot be changed. Pass extra flags with `PYTEST_ADDOPTS`. Vitest rejects this option; use `--vitest-config` and configure options in Vitest. | +| `--command` | Override the default base command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, and Playwright run/discovery. Vitest rejects this option; use `--vitest-config` and configure options in Vitest. | | `--vitest-config` | Vitest config file used by both planning and execution; defaults to Vitest config discovery. See [migration instructions](docs/running.md#vitest-integration). | | `--min-parallelism` | Minimum CI node or worker count DDTest considers when planning. | | `--max-parallelism` | Maximum CI node or worker count DDTest considers when planning. | diff --git a/docs/settings.md b/docs/settings.md index fb3ef573..117977f5 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -10,7 +10,7 @@ platforms or frameworks require explicit selection. | --- | --- | --- | ---: | --- | | `--platform` | `DD_TEST_OPTIMIZATION_RUNNER_PLATFORM` | | auto-detected | Language/platform. Currently supported: `ruby`, `python`, `javascript`. | | `--framework` | `DD_TEST_OPTIMIZATION_RUNNER_FRAMEWORK` | | auto-detected | Test framework. Currently supported: `rspec`, `minitest`, `pytest`, `cucumber`, `cypress`, `jest`, `mocha`, `playwright`, `vitest`. | -| `--command` | `DD_TEST_OPTIMIZATION_RUNNER_COMMAND` | | `""` | Override the default base test command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, and Playwright run/discovery, and pytest run/discovery (since 1.7.0). DDTest appends selected tests and framework-specific flags. For ddtest versions prior to 1.7.0 with pytest, the command cannot be changed. Pass extra flags with `PYTEST_ADDOPTS`. Vitest rejects this option; use `--vitest-config` and configure options in Vitest. | +| `--command` | `DD_TEST_OPTIMIZATION_RUNNER_COMMAND` | | `""` | Override the default base test command for supported framework modes. Used by RSpec and Minitest run/discovery, Cucumber, Cypress, Jest, Mocha, and Playwright run/discovery. DDTest appends selected tests and framework-specific flags. Vitest rejects this option; use `--vitest-config` and configure options in Vitest. | | `--vitest-config` | `DD_TEST_OPTIMIZATION_RUNNER_VITEST_CONFIG` | | `""` | Vitest config file used by both planning and execution; defaults to Vitest config discovery. See [migration instructions](running.md#vitest-integration). | | `--min-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM` | | physical CPU count | Minimum count DDTest considers when planning. Interpret it as CI nodes in CI-node mode, or workers in a single-node run. | | `--max-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM` | | physical CPU count | Maximum count DDTest considers when planning. Interpret it as CI nodes in CI-node mode, or workers in a single-node run. | From 4b69fe75467fb4169f4194ac6dd7d91eeabde1c9 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Thu, 8 Oct 2026 12:20:05 +0200 Subject: [PATCH 10/10] Use Vitest shutdown timeout for leaked handles --- internal/compatibility/vitest_test.go | 78 ++++++++++++++++++++++++--- internal/framework/scripts/vitest.mjs | 4 +- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/internal/compatibility/vitest_test.go b/internal/compatibility/vitest_test.go index c657eb48..8f08e9c4 100644 --- a/internal/compatibility/vitest_test.go +++ b/internal/compatibility/vitest_test.go @@ -26,6 +26,22 @@ func configureVitest(config string) { settings.Get().VitestConfig = config } +// Teardown completes asynchronously, but the interval deliberately stays active. +// This distinguishes a leaked handle from a teardown promise that never resolves. +func writeVitestShutdownSetup(t *testing.T, root string) { + t.Helper() + writeFixture(t, root, "shutdown-setup.mjs", `import { writeFileSync } from 'node:fs' +export default function () { + if (process.env.DDTEST_LEAK_HANDLE !== 'true') return + setInterval(() => {}, 1000) + return async () => { + await new Promise(resolve => setTimeout(resolve, 50)) + writeFileSync(process.env.DDTEST_VITEST_TEARDOWN, 'completed') + } +} +`) +} + func TestVitestAdapterIntegration(t *testing.T) { nodeModules := requireEnv(t, "DDTEST_VITEST_NODE_MODULES") resetSettingsAfterTest(t) @@ -101,6 +117,31 @@ test('must not run', () => { t.Fatalf("Vitest lifecycle = %q, error = %v", contents, err) } + t.Run("completed teardown with a leaked handle", func(t *testing.T) { + writeVitestShutdownSetup(t, root) + writeFixture(t, root, "shutdown.config.mjs", `export default { test: { + include: ['checks/selected.check.js'], setupFiles: ['./setup.js'], + globalSetup: ['./shutdown-setup.mjs'], teardownTimeout: 1000, +} }`) + configureVitest("shutdown.config.mjs") + teardown := filepath.Join(t.TempDir(), "teardown.txt") + // Leave ample startup time, but fail if shutdown waits on the leaked timer. + shutdownCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + err := framework.NewVitest().RunTests(shutdownCtx, []string{"checks/selected.check.js"}, map[string]string{ + "DDTEST_VITEST_WORKER": "selected", "DDTEST_LEAK_HANDLE": "true", "DDTEST_VITEST_TEARDOWN": teardown, + }) + if contents, readErr := os.ReadFile(teardown); readErr != nil || string(contents) != "completed" { + t.Fatalf("global teardown did not complete: contents = %q, error = %v", contents, readErr) + } + if shutdownCtx.Err() != nil { + t.Fatalf("Vitest stayed alive after teardown completed: %v", shutdownCtx.Err()) + } + if err != nil { + t.Fatal(err) + } + }) + t.Run("exact file membership with overlapping names", func(t *testing.T) { writeFixture(t, root, "vitest.overlap.mjs", `export default { test: { dir: 'src', include: ['**/test.js'], setupFiles: ['./setup.js'], @@ -330,7 +371,10 @@ func TestVitestTracingIntegration(t *testing.T) { if err := os.Symlink(nodeModules, filepath.Join(root, "node_modules")); err != nil { t.Fatal(err) } - writeFixture(t, root, "vitest.config.mjs", `export default { test: { dir: 'src', include: ['**/test.js'] } }`) + writeVitestShutdownSetup(t, root) + writeFixture(t, root, "vitest.config.mjs", `export default { test: { + dir: 'src', include: ['**/test.js'], globalSetup: ['./shutdown-setup.mjs'], teardownTimeout: 1000, +} }`) writeFixture(t, root, "src/endOfYear/test.js", `import { test } from 'vitest' test('ddtest exact traced assignment', () => { if (process.env.DDTEST_FAIL_ASSIGNED === 'true') throw new Error('assigned failure') @@ -380,17 +424,37 @@ test('ddtest unassigned traced file', () => { throw new Error('unassigned file r "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "false", "DD_TRACE_STARTUP_LOGS": "false", }) - ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) - defer cancel() - for _, failure := range []bool{false, true} { - t.Run("assigned failure="+strconv.FormatBool(failure), func(t *testing.T) { + for _, tc := range []struct { + name string + failure bool + leakedHandle bool + }{ + {name: "passing"}, + {name: "failing", failure: true}, + {name: "passing with leaked handle", leakedHandle: true}, + {name: "failing with leaked handle", failure: true, leakedHandle: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + teardown := filepath.Join(t.TempDir(), "teardown.txt") mu.Lock() payloads = nil mu.Unlock() err := vitest.RunTests(ctx, []string{"src/endOfYear/test.js"}, map[string]string{ - "DDTEST_FAIL_ASSIGNED": strconv.FormatBool(failure), + "DDTEST_FAIL_ASSIGNED": strconv.FormatBool(tc.failure), + "DDTEST_LEAK_HANDLE": strconv.FormatBool(tc.leakedHandle), + "DDTEST_VITEST_TEARDOWN": teardown, }) - if failure { + if tc.leakedHandle { + if contents, readErr := os.ReadFile(teardown); readErr != nil || string(contents) != "completed" { + t.Fatalf("global teardown did not complete: contents = %q, error = %v", contents, readErr) + } + } + if ctx.Err() != nil { + t.Fatalf("Vitest did not exit before the watchdog: %v", ctx.Err()) + } + if tc.failure { if err == nil || !strings.Contains(err.Error(), "exit status 1") { t.Fatalf("expected assigned-test failure, got %v", err) } diff --git a/internal/framework/scripts/vitest.mjs b/internal/framework/scripts/vitest.mjs index 62c6d1a4..396264df 100644 --- a/internal/framework/scripts/vitest.mjs +++ b/internal/framework/scripts/vitest.mjs @@ -40,5 +40,7 @@ try { } } } finally { - await context.close() + // Use Vitest's CLI shutdown policy: close gracefully, then terminate after + // teardownTimeout if leaked handles keep the process alive. + await context.exit() }