From 4939e1d17cd132eaa46bef34f487db51b1fd30b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 30 Sep 2026 16:37:54 +0200 Subject: [PATCH 1/2] Prototype onboarding validation and agent evaluations Combine the reviewed PR #152 implementation: CI scope and tracer checks, paired Jest and Vitest validation, isolated feature scenarios and configuration preparation, compact evidence alongside the upstream HTML report, and repeatable agent evaluations. --- README.md | 131 ++++- docs/design/agent-driven-onboarding.md | 18 +- go.mod | 3 +- go.sum | 4 + internal/cmd/testdrive.go | 7 +- internal/cmd/testdrive_test.go | 6 + .../compatibility/project_environment_test.go | 42 +- internal/framework/command_args.go | 8 +- internal/onboard/class_validator_test.go | 198 +++++++ internal/onboard/composite.go | 150 +++++ internal/onboard/composite_test.go | 97 ++++ internal/onboard/express_validator_test.go | 203 +++++++ internal/onboard/instructions/github.md | 53 +- internal/onboard/instructions/vitest.md | 49 ++ internal/onboard/jest_command.go | 119 ++++ internal/onboard/jest_forwarding.go | 170 ++++++ internal/onboard/jest_forwarding_test.go | 115 ++++ internal/onboard/multiframework_test.go | 1 - internal/onboard/npm_prefix.go | 54 ++ internal/onboard/onboard.go | 198 ++++--- internal/onboard/onboard_test.go | 22 +- internal/onboard/packaging.go | 68 +++ internal/onboard/release.go | 63 +++ internal/onboard/runtime.go | 378 +++++++++++++ internal/onboard/runtime_bootstrap.go | 144 +++++ internal/onboard/runtime_bootstrap_test.go | 125 +++++ internal/onboard/runtime_conditions.go | 353 ++++++++++++ internal/onboard/runtime_conditions_test.go | 137 +++++ internal/onboard/runtime_node.go | 181 ++++++ internal/onboard/runtime_node_test.go | 92 ++++ internal/onboard/runtime_test.go | 233 ++++++++ internal/onboard/runtime_values.go | 209 +++++++ internal/onboard/scope.go | 195 +++++++ internal/onboard/scope_test.go | 238 ++++++++ internal/onboard/scripts.go | 516 ++++++++++++++++++ internal/onboard/scripts_files.go | 69 +++ internal/onboard/scripts_files_test.go | 71 +++ internal/onboard/scripts_test.go | 346 ++++++++++++ internal/onboard/shell_metadata.go | 146 +++++ internal/onboard/shell_sequence.go | 81 +++ internal/onboard/tinypool_test.go | 109 ++++ internal/onboard/variants.go | 181 ++++++ internal/onboard/variants_test.go | 117 ++++ internal/onboard/vitest_test.go | 139 +++++ internal/platform/javascript_selection.go | 121 ++++ .../platform/javascript_selection_test.go | 98 ++++ internal/platform/javascript_test.go | 31 ++ internal/testdrive/build_prerequisites.go | 91 +++ .../testdrive/build_prerequisites_test.go | 105 ++++ internal/testdrive/cleanup_test.go | 237 ++++++++ internal/testdrive/command_selection_test.go | 107 ++++ internal/testdrive/configuration_workspace.go | 131 +++++ .../testdrive/configuration_workspace_test.go | 34 ++ internal/testdrive/configurations.go | 267 +++++++++ internal/testdrive/configurations_test.go | 140 +++++ .../testdrive/frameworks_integration_test.go | 60 +- internal/testdrive/intake/events.go | 38 ++ internal/testdrive/intake/findings.go | 20 + internal/testdrive/intake/server.go | 7 +- internal/testdrive/intake/settings.go | 70 ++- internal/testdrive/intake/settings_test.go | 62 ++- internal/testdrive/javascript.go | 11 +- internal/testdrive/jest.go | 413 ++++++++++++++ internal/testdrive/jest_coverage.go | 49 ++ .../jest_coverage_integration_test.go | 140 +++++ internal/testdrive/jest_features.go | 338 ++++++++++++ internal/testdrive/jest_preflight.go | 176 ++++++ internal/testdrive/jest_preflight_test.go | 169 ++++++ internal/testdrive/jest_projects.go | 92 ++++ .../jest_projects_integration_test.go | 114 ++++ internal/testdrive/jest_projects_test.go | 90 +++ internal/testdrive/jest_recommendation.go | 132 +++++ .../testdrive/jest_recommendation_test.go | 115 ++++ internal/testdrive/jest_skipping.go | 65 +++ internal/testdrive/jest_test.go | 396 ++++++++++++++ internal/testdrive/multiframework_test.go | 15 +- internal/testdrive/repeatability.go | 92 ++++ internal/testdrive/repeatability_test.go | 65 +++ internal/testdrive/report.html | 2 +- internal/testdrive/retained_execution.go | 119 ++++ internal/testdrive/retained_execution_test.go | 165 ++++++ internal/testdrive/scripts/vitest_config.mjs | 22 + internal/testdrive/session.go | 29 +- internal/testdrive/session_test.go | 63 +-- internal/testdrive/testdrive.go | 353 ++++++++---- .../testdrive/testdrive_integration_test.go | 24 +- internal/testdrive/testdrive_test.go | 461 +++------------- internal/testdrive/validation.go | 371 +++++++++++++ internal/testdrive/validation_html.go | 52 ++ internal/testdrive/validation_html_test.go | 90 +++ internal/testdrive/validation_summary.go | 315 +++++++++++ internal/testdrive/validation_summary_test.go | 205 +++++++ internal/testdrive/validation_test.go | 176 ++++++ internal/testdrive/vitest.go | 255 +++++++++ internal/testdrive/vitest_test.go | 255 +++++++++ skills/ddtest-onboarding-eval/SKILL.md | 171 ++++++ .../scripts/run_eval.py | 377 +++++++++++++ .../scripts/test_run_eval.py | 251 +++++++++ 98 files changed, 12954 insertions(+), 732 deletions(-) create mode 100644 internal/onboard/class_validator_test.go create mode 100644 internal/onboard/composite.go create mode 100644 internal/onboard/composite_test.go create mode 100644 internal/onboard/express_validator_test.go create mode 100644 internal/onboard/instructions/vitest.md create mode 100644 internal/onboard/jest_command.go create mode 100644 internal/onboard/jest_forwarding.go create mode 100644 internal/onboard/jest_forwarding_test.go create mode 100644 internal/onboard/npm_prefix.go create mode 100644 internal/onboard/packaging.go create mode 100644 internal/onboard/release.go create mode 100644 internal/onboard/runtime.go create mode 100644 internal/onboard/runtime_bootstrap.go create mode 100644 internal/onboard/runtime_bootstrap_test.go create mode 100644 internal/onboard/runtime_conditions.go create mode 100644 internal/onboard/runtime_conditions_test.go create mode 100644 internal/onboard/runtime_node.go create mode 100644 internal/onboard/runtime_node_test.go create mode 100644 internal/onboard/runtime_test.go create mode 100644 internal/onboard/runtime_values.go create mode 100644 internal/onboard/scope.go create mode 100644 internal/onboard/scope_test.go create mode 100644 internal/onboard/scripts.go create mode 100644 internal/onboard/scripts_files.go create mode 100644 internal/onboard/scripts_files_test.go create mode 100644 internal/onboard/scripts_test.go create mode 100644 internal/onboard/shell_metadata.go create mode 100644 internal/onboard/shell_sequence.go create mode 100644 internal/onboard/tinypool_test.go create mode 100644 internal/onboard/variants.go create mode 100644 internal/onboard/variants_test.go create mode 100644 internal/onboard/vitest_test.go create mode 100644 internal/platform/javascript_selection.go create mode 100644 internal/platform/javascript_selection_test.go create mode 100644 internal/testdrive/build_prerequisites.go create mode 100644 internal/testdrive/build_prerequisites_test.go create mode 100644 internal/testdrive/cleanup_test.go create mode 100644 internal/testdrive/command_selection_test.go create mode 100644 internal/testdrive/configuration_workspace.go create mode 100644 internal/testdrive/configuration_workspace_test.go create mode 100644 internal/testdrive/configurations.go create mode 100644 internal/testdrive/configurations_test.go create mode 100644 internal/testdrive/jest.go create mode 100644 internal/testdrive/jest_coverage.go create mode 100644 internal/testdrive/jest_coverage_integration_test.go create mode 100644 internal/testdrive/jest_features.go create mode 100644 internal/testdrive/jest_preflight.go create mode 100644 internal/testdrive/jest_preflight_test.go create mode 100644 internal/testdrive/jest_projects.go create mode 100644 internal/testdrive/jest_projects_integration_test.go create mode 100644 internal/testdrive/jest_projects_test.go create mode 100644 internal/testdrive/jest_recommendation.go create mode 100644 internal/testdrive/jest_recommendation_test.go create mode 100644 internal/testdrive/jest_skipping.go create mode 100644 internal/testdrive/jest_test.go create mode 100644 internal/testdrive/repeatability.go create mode 100644 internal/testdrive/repeatability_test.go create mode 100644 internal/testdrive/retained_execution.go create mode 100644 internal/testdrive/retained_execution_test.go create mode 100644 internal/testdrive/scripts/vitest_config.mjs create mode 100644 internal/testdrive/validation.go create mode 100644 internal/testdrive/validation_html.go create mode 100644 internal/testdrive/validation_html_test.go create mode 100644 internal/testdrive/validation_summary.go create mode 100644 internal/testdrive/validation_summary_test.go create mode 100644 internal/testdrive/validation_test.go create mode 100644 internal/testdrive/vitest.go create mode 100644 internal/testdrive/vitest_test.go create mode 100644 skills/ddtest-onboarding-eval/SKILL.md create mode 100644 skills/ddtest-onboarding-eval/scripts/run_eval.py create mode 100644 skills/ddtest-onboarding-eval/scripts/test_run_eval.py diff --git a/README.md b/README.md index 65b61a15..4f5ac782 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,9 @@ commands, asks for confirmation, and runs your own tests against a local intake. It needs no Datadog account, API key, or Agent. Non-interactive callers can review the preview, then use `ddtest testdrive --yes`. -All nine frameworks listed above are supported. If a repository contains several +All nine frameworks listed above can collect local telemetry. Compatibility and +feature validation currently support Jest; other frameworks explicitly remain +unvalidated. If a repository contains several runners, select one with `--framework`. Use `--command` to select a custom entry point or a small representative part of a large suite: @@ -38,22 +40,95 @@ ddtest onboard --framework playwright ddtest testdrive --framework playwright --command 'npm run test:e2e -- --project=chromium' --yes ``` -The terminal links to a self-contained HTML report, decoded JSON traffic, and -complete test output under `.testoptimization/testdrive//`. Reports -separate instrumentation success from failed tests, and explicitly indicate when -coverage was not reported. Tracer configuration errors are shown separately. +Testdrive prints a terminal summary and retains two reports: +`.testoptimization/report.html` uses the upstream HTML renderer to show the latest +instrumented suite's tests, findings, attempts, source excerpts, coverage and full +command output. With `--all`, output is labeled by configuration. +`.testoptimization/testdrive.json` records the validation verdict: an explicit +`success` flag, compatibility, CI runtime and feature verdicts, and the working +directory. Each validation run records its exact shell-quoted command, +instrumentation/probe mode, exit code and aggregate counts. Mismatch examples are +limited to ten and diagnostic text is capped at 1,024 characters in the JSON. + +The HTML describes the instrumented suite, not the overall validation verdict; +synthetic feature probes never replace its findings. It links to the compact +JSON rather than discarded traffic or output files. Each executed suite replaces +the HTML at the same path. Configuration-only checks leave earlier HTML unchanged +and do not claim that tests were rerun. Raw output and telemetry payload files +are discarded after validation; report directories do not accumulate. + +JavaScript/Python fallback tracers, captured requests, and runner files live in a +private OS temporary directory. Ruby fallback installation uses the project bundle +and retains its dependency changes. Testdrive removes that directory on success, failure, and +handled cancellation. Temporary probe files inside the project are also removed. +Bounded setup errors and summaries of attempted runs are preserved when the report +can be written. Other `.testoptimization` data is preserved. + +Jest validation has two stages: + +1. Run the same command without instrumentation and with reporting-only + instrumentation. Compare test identities, outcomes, failure details, and + process exit codes; ignore timings, result ordering, and stack frames. + Existing failures are acceptable when both runs match. If results differ, + repeat the pair: changing results are inconclusive, while a repeatable + difference is a suspected regression. Matching results require matching + telemetry before compatibility can be reported. Setup failures before tests + execute are inconclusive. +2. Place a temporary probe beside an existing test, preserving the project's + Jest configuration. Establish passing and failing controls, then enable one + feature per run: auto retries, early flake detection, skipping, quarantine, + disabled tests, and attempt-to-fix. Check actual execution, exit outcome, and + feature-specific telemetry. Remove the probe when finished, including on + errors. If the command/configuration cannot select the probe, its checks are + inconclusive. These checks validate the probe under the selected configuration; + they do not establish compatibility for every test environment in a monorepo. + +Jest skipping uses the control's `test.source.file` to identify the file to skip; +test-management scenarios keep the reported suite and test names. Some tracer +versions report a different suite name when a file is skipped. The report retains +both names and notes the difference separately from whether skipping worked. +A missing source path makes the skipping check inconclusive. + +`testdrive.json` separates compatibility (`compatible`, `suspected regression`, +`inconclusive`) from each feature (`passed`, `failed`, `inconclusive`, `unvalidated`). +For Jest repositories with detected GitHub Actions test jobs, `ci_runtime` records +whether each instrumented Node runtime satisfies its workflow-selected tracer's +`engines.node` requirement. The checker reads public GitHub action metadata at the +configured action ref and npm package metadata, honoring `js-tracer-version` when +set. No tracer version or minimum Node version is pinned in the checker. This can +differ from the tracer installed for the local testdrive. + +The check supports literal `actions/setup-node` versions, static matrix axes, and +static `include`/`exclude`, and matrix equality/inequality, boolean values, `startsWith`, `!`, `&&`, `||`, and parentheses. +Explicitly excluded entries remain visible as uninstrumented. Dynamic matrices, +version files, LTS aliases, other conditions, mixed-type comparisons, unsupported engine +ranges, and unavailable metadata are inconclusive. Currently engine comparison +supports minimum requirements such as `>=22` or `>=22.2.0`; it does not approximate +other ranges. This checks the declared setup-node runtime, not arbitrary shell +commands that might later change Node. It does not execute CI or prove bootstrap +correctness, remote instrumentation, or Datadog backend connectivity. + +The command exits successfully only when local compatibility, all feature checks, +and the applicable CI runtime check succeed. A runtime incompatibility or unknown +configuration makes `success` false even if every local test matches. Repositories +without detected GitHub Actions test jobs can still pass local validation; their +CI runtime result is explicitly `not applicable`. The suite's own failures do not +automatically fail validation. +Other frameworks collect reporting-only telemetry and return an inconclusive +validation result until adapters exist. + The local intake supports agentless traffic; Agent/EVP routing is not supported. -Receiving events does not verify test skipping, EFD, or Test Management behavior. -Keep this directory out of source control. Each run has its own files and loopback port. +No results are sent to Datadog. Temporary installations and run files are removed; +only the HTML findings and compact JSON validation reports remain. Keep both out of source control. Testdrive reuses the project's tracer when the platform's tracer check succeeds. -If the check fails, it attempts to install the latest release inside the session. +If the check fails, it attempts a fallback installation (latest by default). `--tracer-version` selects a release or Git revision for that fallback installation; JavaScript and Python installations leave project dependency files unchanged; Ruby uses `bundle add datadog-ci`, which updates the project Gemfile and lockfile: ```sh -ddtest testdrive --tracer-version 6.15.0 --yes # JavaScript example +ddtest testdrive --tracer-version --yes ddtest testdrive --tracer-version 'git:' --yes ``` @@ -76,9 +151,43 @@ Local testdrive prerequisites: first, or use its existing test command that manages them. Testdrive does not install browsers or start applications on its own. +Jest preflight resolves the fallback selection once before installation and inspects +`--showConfig` for the effective Jest version, runner, and configuration. It checks +known dd-trace 5/6 Jest requirements and the tracer's Node engine minimum before +running the suite. Unknown combinations remain unverified. Use `--command` with +the project's actual Jest arguments when it uses a custom config. Preflight loads +project JavaScript after confirmation. Preview runs a read-only installed-tracer probe +(with a 10-second timeout) to show whether the tracer will be reused or installed; +it does not load Jest configuration, install dependencies, or run tests. + +`ddtest testdrive --check-only --yes` performs those configuration checks and static +CI checks without installing a tracer or running tests. It updates the same JSON +report, with `check_only: true`, `checks_passed`, and test execution marked not +exercised. A successful configuration check does not set validation `success`. +The latest paired Jest execution is preserved under `retained_execution.result`, +including its timestamp, tracer, commands, counts, and feature verdicts. Later +configuration checks, unsupported-framework runs, and setup failures retain this +historical evidence; a new paired Jest execution supersedes it. There is no growing +report history. Retained evidence never changes the current invocation's verdict: +after changing the command, configuration, source, dependencies, runtime, or tracer, +rerun full validation before claiming current compatibility or feature success. + +Full validation records `local_success` separately from the combined `success`. +CI Node compatibility and exact local/CI tracer agreement are separate findings; +actual CI execution is always `not exercised`. Moving CI selectors are valid only +for the version resolved during this check. Pin the selected release in the +workflow to retain that agreement. Unsupported checker syntax requires review, +not a workflow rewrite. Manual review does not override a programmatic verdict. Onboarding uses +`datadog/test-visibility-github-action@v3` and leaves tracer-version inputs unset. + Project dependency manifests and lockfiles are not edited by testdrive. Testdrive -uses the framework’s normal command; pass `--command` to run a package script and -its lifecycle hooks or other custom setup. Tracer downloads require network access. This release covers root projects and GitHub Actions onboarding; +automatically selects a unique, statically resolved root-level Jest CI command, +including its package script and options. Without a CI command, it tries `test:ci` +and then `test`; otherwise it uses the runner default. Ambiguous commands, lifecycle +hooks, and scripts that cannot forward Jest options safely require review and an +explicit `--command`. An explicit command always takes precedence. Generated Jest +coverage goes into temporary session storage and is removed afterward; existing +customer coverage is preserved. Tracer downloads require network access. This release covers root projects and GitHub Actions onboarding; monorepo orchestration and other CI providers are outside this scope. See the [Milestone 2 validation record](docs/testing/onboarding-milestone-2.md) diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md index 9e013d7d..fb3112dd 100644 --- a/docs/design/agent-driven-onboarding.md +++ b/docs/design/agent-driven-onboarding.md @@ -2,7 +2,21 @@ Status: Milestones 0, 1, and 2 implemented; Milestones 3 and 4 proposed -Last updated: 2026-09-22 +Last updated: 2026-09-24 + +## Validation prototype update + +The prototype based on PR #147 now uses the validation contract described in +[the README](../../README.md): paired Jest compatibility runs, separate controlled +feature probes, terminal output, the upstream HTML findings report at +`.testoptimization/report.html` and compact validation evidence at +`.testoptimization/testdrive.json`, and project-tracer reuse or a resolved fallback installation. +The compact report retains success, verdicts, validation commands, modes, exit codes, aggregate counts and bounded diagnostics. The HTML embeds full instrumented command output; separate raw output and event files are discarded. Each run removes its scratch files and updates the same reports. Configuration-only, unsupported-framework, and setup-failure runs preserve the latest paired Jest execution as historical evidence, without reusing its verdict for the current invocation. A new paired Jest execution supersedes that evidence. Onboarding targets the v3 GitHub Action without tracer +version pins in the universal template. The agent selects a compatible release for the repository and aligns the CI input with the locally checked version. Jest preflight checks the effective configuration before suite execution; `--check-only` reruns configuration checks without tests. Static matrices support includes/excludes and common boolean conditions. Unknown checker syntax remains unverified and must not cause workflow rewrites. Local compatibility, features, CI runtime compatibility, tracer agreement, and actual CI execution are reported separately. Other frameworks can collect telemetry but remain explicitly +unvalidated. Receiving events alone is not proof of compatibility. + +The milestone narrative below describes the earlier implementation and its +historical evidence, including raw traffic retention and pinned versions. The upstream HTML renderer is retained unchanged; its current artifact link points to the compact validation JSON because raw files are cleaned up. ## Goal @@ -99,7 +113,7 @@ Milestone 1 turned the spike into the current `onboard` and `testdrive` flow des Its important interaction contract is: -- detection and preview happen before any write or external command; +- detection and preview happen before any write, installation, or test execution; a read-only tracer probe selects the reuse or installation preview; - `ddtest testdrive --yes` is the explicit non-interactive path; - running the command is one decision—there is no persisted plan, checksum, approval file, or second execution command; - instrumentation success is independent of whether customer tests pass; diff --git a/go.mod b/go.mod index 606ac794..f241c53f 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,10 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.12.1 github.com/tinylib/msgp v1.6.5 + go.yaml.in/yaml/v3 v3.0.5 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 - go.yaml.in/yaml/v3 v3.0.5 + mvdan.cc/sh/v3 v3.14.1 ) require ( diff --git a/go.sum b/go.sum index 47b57378..117b09eb 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= +github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -55,3 +57,5 @@ golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +mvdan.cc/sh/v3 v3.14.1 h1:bXkhQWNHCs0KZEChF8hYS6FC+T2N9mUZLbQv9blditI= +mvdan.cc/sh/v3 v3.14.1/go.mod h1:syYCoFET8w9tvevxiXUtY8/ICrU+l26jHmhJDra3Vwo= diff --git a/internal/cmd/testdrive.go b/internal/cmd/testdrive.go index 25b5ec93..c1c834aa 100644 --- a/internal/cmd/testdrive.go +++ b/internal/cmd/testdrive.go @@ -22,7 +22,7 @@ func newTestdriveCommand() *cobra.Command { command := &cobra.Command{ Use: "testdrive", Short: "Try Test Optimization on the local test suite", - Long: "Runs the detected test suite once with Datadog Test Optimization and a local intake. No Datadog API key is required.", + Long: "Validates Jest/Vitest compatibility with paired runs and controlled feature scenarios against a local intake. Other frameworks collect telemetry but remain unvalidated. No Datadog API key is required.", Args: func(cmd *cobra.Command, args []string) error { if err := cobra.NoArgs(cmd, args); err != nil { return err @@ -32,10 +32,13 @@ func newTestdriveCommand() *cobra.Command { }, } var version string + var checkOnly, all bool + command.Flags().BoolVar(&all, "all", false, "Run known chained build prerequisites, validate every discovered Jest/Vitest CI command, and aggregate one report pair") + command.Flags().BoolVar(&checkOnly, "check-only", false, "Check Jest/Vitest and static CI configuration without installing a tracer or running tests") command.Flags().StringVar(&version, "tracer-version", "latest", "Fallback tracer release or git:, used only when the project has no tracer") command.Flags().Bool("yes", false, "Run after printing the changes and commands") command.RunE = func(cmd *cobra.Command, _ []string) error { - execution, err := testdrive.Prepare(version) + execution, err := testdrive.Prepare(version, checkOnly, all) if err != nil { return err } diff --git a/internal/cmd/testdrive_test.go b/internal/cmd/testdrive_test.go index 9f1d78e4..0b9da7bc 100644 --- a/internal/cmd/testdrive_test.go +++ b/internal/cmd/testdrive_test.go @@ -133,3 +133,9 @@ func TestTestdriveCommandPreview(t *testing.T) { }) } } + +func TestTestdriveCommandHasCheckOnlyFlag(t *testing.T) { + if testdriveCmd.Flags().Lookup("check-only") == nil { + t.Fatal("testdrive command does not define --check-only") + } +} diff --git a/internal/compatibility/project_environment_test.go b/internal/compatibility/project_environment_test.go index 5e71134f..f010485a 100644 --- a/internal/compatibility/project_environment_test.go +++ b/internal/compatibility/project_environment_test.go @@ -3,6 +3,7 @@ package compatibility import ( "bytes" "context" + "encoding/json" "os" "os/exec" "path/filepath" @@ -44,7 +45,7 @@ func TestJavaScriptProjectEnvironment(t *testing.T) { defer cancel() root := filepath.Join(t.TempDir(), "project with spaces") writeFixture(t, root, "package.json", `{"name":"pnp-regression","private":true,"scripts":{"test":"jest"},"dependencies":{"dd-trace":"file:./tracer"}}`) - writeFixture(t, root, "tracer/package.json", `{"name":"dd-trace","version":"1.0.0","dependencies":{"pnp-tracer-helper":"file:../helper"}}`) + writeFixture(t, root, "tracer/package.json", `{"name":"dd-trace","version":"6.18.0","engines":{"node":">=18"},"dependencies":{"pnp-tracer-helper":"file:../helper"}}`) writeFixture(t, root, "helper/package.json", `{"name":"pnp-tracer-helper","version":"1.0.0","main":"index.js"}`) writeFixture(t, root, "helper/index.js", "module.exports = 'loaded through PnP';\n") writeFixture(t, root, "tracer/ci/init.js", "global.ddtestTracer = require('pnp-tracer-helper');\n") @@ -88,10 +89,20 @@ process.on('exit', () => { writeFixture(t, root, "example.test.js", "// Worker-startup fixture.\n") writeFixture(t, root, "worker.cjs", `const assert = require('assert'); const fs = require('fs'); -if (process.argv.includes('--listTests')) { +if (process.argv.includes('--showConfig')) { + assert.strictEqual(global.ddtestTracer, undefined); + console.log(JSON.stringify({ version: '30.2.0', configs: [{ rootDir: process.cwd(), testRunner: 'jest-circus/runner' }] })); +} else if (process.argv.includes('--listTests')) { assert.strictEqual(global.ddtestTracer, undefined); assert(process.argv.includes('--json')); console.log(JSON.stringify([require('path').resolve('example.test.js')])); +} else if (process.argv.includes('--outputFile')) { + const instrumented = process.env.DD_CIVISIBILITY_ENABLED === 'true'; + assert.strictEqual(global.ddtestTracer, instrumented ? 'loaded through PnP' : undefined); + fs.appendFileSync('worker-ran', instrumented ? 'instrumented\n' : 'baseline\n'); + fs.writeFileSync(process.argv[process.argv.indexOf('--outputFile') + 1], JSON.stringify({ + testResults: [{ name: require('path').resolve('example.test.js'), assertionResults: [{ fullName: 'works', status: 'passed' }] }] + })); } else { assert.strictEqual(global.ddtestTracer, 'loaded through PnP'); fs.writeFileSync('worker-ran', 'instrumented'); @@ -113,10 +124,29 @@ if (process.argv.includes('--listTests')) { require.NoError(t, err) var output bytes.Buffer err = drive.Run(ctx, &output) - // This fixture checks startup, not telemetry. A successful command with - // no events is distinguishable from a failed Node preload. - require.ErrorContains(t, err, "command exited successfully, but Test Optimization sent no test events", output.String()) - require.FileExists(t, filepath.Join(root, "worker-ran")) + // The fixture exercises both startup modes but emits no telemetry, so + // successful worker startup must not produce a passing validation verdict. + require.ErrorContains(t, err, "validation is incomplete", output.String()) + data, err := os.ReadFile(filepath.Join(root, "worker-ran")) + require.NoError(t, err) + require.Contains(t, string(data), "baseline\ninstrumented\n") + data, err = os.ReadFile(filepath.Join(root, ".testoptimization", "testdrive.json")) + require.NoError(t, err) + var report struct { + Success bool + Runs []struct { + Name string + ExitCode *int `json:"exit_code"` + } + } + require.NoError(t, json.Unmarshal(data, &report)) + require.False(t, report.Success) + require.GreaterOrEqual(t, len(report.Runs), 2) + for i, name := range []string{"baseline", "reporting-only"} { + require.Equal(t, name, report.Runs[i].Name) + require.NotNil(t, report.Runs[i].ExitCode) + require.Zero(t, *report.Runs[i].ExitCode) + } }) } diff --git a/internal/framework/command_args.go b/internal/framework/command_args.go index d1f52cfb..4200f191 100644 --- a/internal/framework/command_args.go +++ b/internal/framework/command_args.go @@ -6,9 +6,9 @@ import ( "strings" ) -// frameworkSeparator finds the framework's end-of-options marker, skipping a +// FrameworkSeparator finds the framework's end-of-options marker, skipping a // wrapper's marker before the framework executable (for example npx -- jest). -func frameworkSeparator(command string, args []string, executable string) int { +func FrameworkSeparator(command string, args []string, executable string) int { start := 0 if frameworkExecutableName(command) != executable { for i, arg := range args { @@ -28,7 +28,7 @@ func frameworkSeparator(command string, args []string, executable string) int { // Framework-generated options must precede its end-of-options marker. func withFrameworkOptions(command string, args []string, executable string, options ...string) []string { args = slices.Clone(args) - if index := frameworkSeparator(command, args, executable); index >= 0 { + if index := FrameworkSeparator(command, args, executable); index >= 0 { return slices.Insert(args, index, options...) } return append(args, options...) @@ -37,7 +37,7 @@ func withFrameworkOptions(command string, args []string, executable string, opti // Everything after a framework's -- is positional, so replace that explicit // selection with the selected files. Leave options before it untouched. func withFrameworkFiles(command string, args []string, executable string, files []string) []string { - if index := frameworkSeparator(command, args, executable); index >= 0 { + if index := FrameworkSeparator(command, args, executable); index >= 0 { args = args[:index+1] } return append(slices.Clone(args), files...) diff --git a/internal/onboard/class_validator_test.go b/internal/onboard/class_validator_test.go new file mode 100644 index 00000000..9096eddf --- /dev/null +++ b/internal/onboard/class_validator_test.go @@ -0,0 +1,198 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const classValidatorWorkflow = `jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: {node: ['lts/*', 'current']} + steps: + - uses: actions/setup-node@v3 + with: {node-version: '${{ matrix.node }}'} + - uses: datadog/test-visibility-github-action@v3 + with: {languages: js, js-tracer-version: '6.17.0'} + - run: npm run test:ci + env: {NODE_OPTIONS: '-r ${{ env.DD_TRACE_PACKAGE }}'} + - run: jq 'del(.devDependencies) | del(.scripts)' package.json > build/package.json + - run: npm publish ./build +` + +const nodeDistributionFixture = `[ + {"version":"v26.2.0","files":["linux-x64","win-x64-exe"]}, + {"version":"v26.4.0","files":["linux-x64"]}, + {"version":"v24.8.0","files":["linux-arm64","osx-arm64-tar"]}, + {"version":"v28.0.0-rc.1","files":["linux-x64"]} +]` + +func TestClassValidatorCIRuntimesAndCommand(t *testing.T) { + root := newJestRepository(t, classValidatorWorkflow) + writeScripts(t, root, map[string]string{"test": "jest", "test:ci": "jest --runInBand --no-cache --coverage --verbose"}) + calls := map[string]int{} + client := &http.Client{Transport: runtimeTransport(func(r *http.Request) (*http.Response, error) { + calls[r.URL.String()]++ + data := nodeDistributionFixture + if r.URL.String() == nodeVersionsManifest { + data = ltsManifestFixture + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(data))}, nil + })} + result := checkCIRuntimes(t.Context(), root, setupNodeResolver(client), func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.17.0", Node: ">=22"}, nil + }) + require.Equal(t, "compatible", result.Status, result) + require.Len(t, result.Jobs, 2) + require.Len(t, result.Review, 2) + require.Equal(t, map[string]int{nodeVersionsManifest: 1, nodeDistributionIndex: 1}, calls) + require.Equal(t, "24", result.Jobs[0].NodeResolution.Version) + require.Equal(t, "26.4.0", result.Jobs[1].NodeResolution.Version) + require.Equal(t, "linux-x64", result.Jobs[1].NodeResolution.Platform) + require.Equal(t, nodeDistributionIndex, result.Jobs[1].NodeResolution.Source) + require.False(t, result.Jobs[1].NodeResolution.ResolvedAt.IsZero()) + command, err := JestValidationCommand(root) + require.NoError(t, err) + require.Equal(t, "npm run test:ci", command) + discovery, err := findWorkflows(root, "javascript", "jest") + require.NoError(t, err) + require.Empty(t, discovery.Unresolved) +} + +func TestLatestNodeAliasesAndUnavailableMetadata(t *testing.T) { + for _, platform := range []string{"linux-x64", "win-x64-exe", "osx-arm64-tar"} { + calls := 0 + client := &http.Client{Transport: runtimeTransport(func(r *http.Request) (*http.Response, error) { + calls++ + require.Equal(t, nodeDistributionIndex, r.URL.String()) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(nodeDistributionFixture))}, nil + })} + resolve := setupNodeResolver(client) + for _, alias := range []string{"current", "latest", "node"} { + result, err := resolve(t.Context(), alias, platform) + require.NoError(t, err) + require.Equal(t, map[string]string{"linux-x64": "26.4.0", "win-x64-exe": "26.2.0", "osx-arm64-tar": "24.8.0"}[platform], result.Version) + } + require.Equal(t, 1, calls) + } + for _, data := range []string{"invalid", "[]", `[{"version":"28.x","files":["linux-x64"]}]`} { + _, err := resolveLatestNode([]byte(data), "linux-x64") + require.Error(t, err) + } + calls := 0 + resolve := setupNodeResolver(&http.Client{Transport: runtimeTransport(func(*http.Request) (*http.Response, error) { calls++; return nil, fmt.Errorf("offline") })}) + for range 2 { + _, err := resolve(t.Context(), "current", "linux-x64") + require.ErrorContains(t, err, "offline") + } + require.Equal(t, 1, calls) + _, err := resolve(t.Context(), "current", "") + require.ErrorContains(t, err, "platform/architecture") +} + +func TestNodeDistributionPlatform(t *testing.T) { + for _, tc := range []struct { + runner any + arch, expected string + }{ + {"ubuntu-latest", "", "linux-x64"}, {"ubuntu-24.04-arm", "", "linux-arm64"}, + {"${{ matrix.os }}", "${{ matrix.arch }}", "win-arm64-exe"}, + {"macos-latest", "arm64", "osx-arm64-tar"}, {"macos-latest", "", ""}, + {[]string{"self-hosted", "linux"}, "", ""}, {"self-hosted", "x64", ""}, + {"ubuntu-latest", "${{ inputs.arch }}", ""}, + } { + require.Equal(t, tc.expected, nodeDistributionPlatform(tc.runner, tc.arch, map[string]any{"os": "windows-2025", "arch": "arm64"})) + } +} + +func TestPackagingCannotHideTestCommandsOrLifecycleHooks(t *testing.T) { + root := t.TempDir() + writeScripts(t, root, map[string]string{"test": "jest"}) + for _, command := range []string{ + `jq 'del(.scripts)' package.json > build/package.json && jest`, + "jq \"$(node tests.js)\" package.json > build/package.json", + `jq '.' package.json > /tmp/output.json`, + `npm publish ./build && jest`, + `npm publish "${TARGET}"`, + } { + result := resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{Run: command}, "javascript", "jest") + require.False(t, result.Review, command) + require.True(t, result.Matched || result.Reason != "", command) + } + for _, hook := range []string{"prepublishOnly", "prepack", "prepare", "postpack", "publish", "postpublish"} { + writeScripts(t, filepath.Join(root, "build"), map[string]string{hook: "jest"}) + result := resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{Run: "npm publish ./build"}, "javascript", "jest") + require.False(t, result.Review) + require.Contains(t, result.Reason, hook) + } +} + +func TestJestCommandSelectionPreservesAliasesAndRejectsAmbiguity(t *testing.T) { + root := newJestRepository(t, classValidatorWorkflow) + writeScripts(t, root, map[string]string{"test": "jest", "test:ci": "npm test -- --coverage --runInBand"}) + command, err := JestValidationCommand(root) + require.NoError(t, err) + require.Equal(t, "npm run test:ci", command) + path := filepath.Join(root, ".github/workflows/test.yml") + require.NoError(t, os.WriteFile(path, []byte(classValidatorWorkflow+" - run: npm test\n"), 0644)) + _, err = JestValidationCommand(root) + require.ErrorContains(t, err, "multiple Jest CI commands") + require.NoError(t, os.WriteFile(path, []byte(classValidatorWorkflow), 0644)) + for _, script := range []string{"jest && jest --config other.js", "node setup.js && jest"} { + writeScripts(t, root, map[string]string{"test:ci": script}) + _, err = JestValidationCommand(root) + require.ErrorContains(t, err, "--command", script) + } +} + +func TestJestCommandSelectionFallbackAndUnsafeCI(t *testing.T) { + root := newJestRepository(t, "jobs: {}") + writeScripts(t, root, map[string]string{"test": "jest", "test:ci": "jest --coverage"}) + command, err := JestValidationCommand(root) + require.NoError(t, err) + require.Equal(t, "npm run test:ci", command) + require.NoError(t, os.WriteFile(filepath.Join(root, "yarn.lock"), nil, 0644)) + command, err = JestValidationCommand(root) + require.NoError(t, err) + require.Equal(t, "yarn run test:ci", command) + workflow := filepath.Join(root, ".github/workflows/test.yml") + for _, command := range []string{"npm run test:ci", "node ci-tests.js"} { + require.NoError(t, os.WriteFile(workflow, []byte("jobs:\n test:\n steps:\n - run: "+command+"\n"), 0644)) + writeScripts(t, root, map[string]string{"test": "jest", "test:ci": "jest --coverage", "pretest:ci": "node setup.js"}) + _, err = JestValidationCommand(root) + require.ErrorContains(t, err, "--command") + } + writeScripts(t, root, map[string]string{"test": "jest"}) + require.NoError(t, os.WriteFile(workflow, []byte("jobs:\n test:\n steps:\n - run: jest\n working-directory: packages/foo\n"), 0644)) + _, err = JestValidationCommand(root) + require.ErrorContains(t, err, "working directory") +} + +func TestUnavailableCurrentMetadataStaysInconclusive(t *testing.T) { + resolve := setupNodeResolver(&http.Client{Transport: runtimeTransport(func(*http.Request) (*http.Response, error) { + return nil, fmt.Errorf("metadata unavailable") + })}) + root := newJestRepository(t, strings.ReplaceAll(classValidatorWorkflow, "'lts/*', ", "")) + writeScripts(t, root, map[string]string{"test:ci": "jest"}) + result := checkCIRuntimes(t.Context(), root, resolve, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.17.0", Node: ">=22"}, nil + }) + require.Equal(t, "inconclusive", result.Status) + require.Len(t, result.Jobs, 1) + require.Nil(t, result.Jobs[0].NodeResolution) + require.Contains(t, result.Jobs[0].Reason, "metadata unavailable") +} diff --git a/internal/onboard/composite.go b/internal/onboard/composite.go new file mode 100644 index 00000000..c741e420 --- /dev/null +++ b/internal/onboard/composite.go @@ -0,0 +1,150 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "fmt" + "maps" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "go.yaml.in/yaml/v3" +) + +func stepNumber(step runtimeStep, index int) int { + if step.Number != 0 { + return step.Number + } + return index + 1 +} + +func stepCondition(step runtimeStep, row map[string]any) (bool, error) { + for _, condition := range append(slices.Clone(step.Parents), step.If) { + active, err := runtimeCondition(condition, row) + if err != nil || !active { + return active, err + } + } + return true, nil +} + +// Only read repository-local composite actions. Preserve ordering and parent +// conditions; never execute an action or pretend to interpret arbitrary JS. +func expandCompositeSteps(root, source string, steps []runtimeStep) []runtimeStep { + if resolved, err := filepath.EvalSymlinks(root); err == nil { + root = resolved + } + remaining := 256 + var expand func(runtimeStep, []string) []runtimeStep + expand = func(step runtimeStep, stack []string) []runtimeStep { + remaining-- + fail := func(err error) []runtimeStep { + step.ResolutionError = fmt.Sprintf("%s: %v", step.Source, err) + return []runtimeStep{step} + } + if remaining < 0 || len(stack) >= 16 { + return fail(fmt.Errorf("local action expansion exceeds its depth or step limit")) + } + if !strings.HasPrefix(step.Uses, "./") { + return []runtimeStep{step} + } + path, err := localActionPath(root, step.Uses) + if err != nil { + return fail(err) + } + if slices.Contains(stack, path) { + return fail(fmt.Errorf("local composite action cycle: %s", step.Uses)) + } + data, err := os.ReadFile(path) + if err != nil { + return fail(err) + } + var action struct { + Inputs map[string]struct{ Default string } `yaml:"inputs"` + Runs struct { + Using string `yaml:"using"` + Steps []runtimeStep `yaml:"steps"` + } `yaml:"runs"` + } + if err := yaml.Unmarshal(data, &action); err != nil { + return fail(err) + } + if action.Runs.Using != "composite" { + return fail(fmt.Errorf("local action %s is not a statically readable composite", step.Uses)) + } + inputs := map[string]string{} + for key, input := range action.Inputs { + inputs[key] = input.Default + } + maps.Copy(inputs, step.With) + relative, _ := filepath.Rel(root, path) + var result []runtimeStep + for i, child := range action.Runs.Steps { + child.Number = step.Number + child.Source = fmt.Sprintf("%s / step %d", filepath.ToSlash(relative), i+1) + child.Parents = append(slices.Clone(step.Parents), step.If) + child.With = maps.Clone(child.With) + for key, value := range child.With { + if match := inputReference.FindStringSubmatch(strings.TrimSpace(value)); match != nil { + if resolved, ok := inputs[match[1]]; ok { + child.With[key] = resolved + } + } + } + env := maps.Clone(step.Env) + if env == nil { + env = map[string]string{} + } + maps.Copy(env, child.Env) + child.Env = env + result = append(result, expand(child, append(slices.Clone(stack), path))...) + if remaining < 0 { + break + } + } + return result + } + var result []runtimeStep + for i, step := range steps { + step.Number = i + 1 + step.Source = fmt.Sprintf("%s / step %d", source, i+1) + result = append(result, expand(step, nil)...) + if remaining < 0 { + break + } + } + return result +} + +var inputReference = regexp.MustCompile(`^\$\{\{\s*inputs\.([a-zA-Z0-9_-]+)\s*\}\}$`) + +func localActionPath(root, uses string) (string, error) { + if !filepath.IsLocal(uses) || strings.Contains(uses, "${{") { + return "", fmt.Errorf("cannot resolve local action path %q", uses) + } + root, err := filepath.EvalSymlinks(root) + if err != nil { + return "", err + } + for _, filename := range []string{"action.yml", "action.yaml"} { + path, err := filepath.EvalSymlinks(filepath.Join(root, uses, filename)) + if os.IsNotExist(err) { + continue + } + if err != nil { + return "", err + } + relative, err := filepath.Rel(root, path) + if err != nil || !filepath.IsLocal(relative) { + return "", fmt.Errorf("local action escapes the repository: %s", uses) + } + return path, nil + } + return "", fmt.Errorf("local action metadata not found: %s", uses) +} diff --git a/internal/onboard/composite_test.go b/internal/onboard/composite_test.go new file mode 100644 index 00000000..334c2a7f --- /dev/null +++ b/internal/onboard/composite_test.go @@ -0,0 +1,97 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const compositeSetup = `inputs: + node: {default: '22'} +runs: + using: composite + steps: + - uses: actions/setup-node@v6 + with: {node-version: '${{ inputs.node }}'} + - run: pnpm install + shell: bash +` + +func writeComposite(t *testing.T, root, name, data string) { + t.Helper() + path := filepath.Join(root, ".github/actions", name, "action.yml") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, []byte(data), 0644)) +} + +func TestLocalCompositeRuntimeAndReactHookFormCommands(t *testing.T) { + workflow := strings.Replace(algorithmsWorkflow, "actions/setup-node@v4\n with: {node-version: '22.x'}", "./.github/actions/install", 1) + workflow += " - run: pnpm build:esm\n - run: pnpm api-extractor:ci\n - run: pnpm bundlewatch\n - run: pnpm e2e\n - run: npx playwright install --with-deps chromium\n" + root := newJestRepository(t, workflow) + writeComposite(t, root, "install", compositeSetup) + writeScripts(t, root, map[string]string{"coverage": "jest", "lint": "eslint .", "build:esm": "rollup -c scripts/rollup.js", "api-extractor:ci": "node scripts/apiExtractor.js", "bundlewatch": "pnpm build:esm && bundlewatch", "e2e": "playwright test"}) + result := checkCIRuntimes(t.Context(), root, nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.17.0", Node: ">=22"}, nil + }) + require.Equal(t, "compatible", result.Status, result) + require.Len(t, result.Jobs, 1) + require.Equal(t, 5, result.Jobs[0].Step) + require.Equal(t, "22", result.Jobs[0].Node) + require.Equal(t, ".github/actions/install/action.yml / step 1", result.Jobs[0].NodeSource) + require.Len(t, result.Review, 5) + discovery, err := findWorkflows(root, "javascript", "jest") + require.NoError(t, err) + require.Empty(t, discovery.Unresolved) + require.Equal(t, discovery.Workflows, discovery.Configured) +} + +func TestCompositeInputsConditionsOrderAndUnknowns(t *testing.T) { + base := strings.Replace(algorithmsWorkflow, "actions/setup-node@v4\n with: {node-version: '22.x'}", "./.github/actions/install", 1) + for _, tc := range []struct{ name, workflow, action, status string }{ + {"default input", base, compositeSetup, "compatible"}, + {"old input", strings.Replace(base, "uses: ./.github/actions/install", "uses: ./.github/actions/install\n with: {node: '20'}", 1), compositeSetup, "incompatible"}, + {"dynamic input", strings.Replace(base, "uses: ./.github/actions/install", "uses: ./.github/actions/install\n with: {node: '${{ inputs.runtime }}'}", 1), compositeSetup, "inconclusive"}, + {"excluded parent", strings.Replace(base, "uses: ./.github/actions/install", "uses: ./.github/actions/install\n if: false", 1), compositeSetup, "inconclusive"}, + {"unknown parent condition", strings.Replace(base, "uses: ./.github/actions/install", "uses: ./.github/actions/install\n if: github.event_name == 'push'", 1), compositeSetup, "inconclusive"}, + {"excluded child", base, strings.Replace(compositeSetup, "uses: actions/setup-node@v6", "uses: actions/setup-node@v6\n if: false", 1), "inconclusive"}, + {"later setup wins", strings.Replace(base, " - run: npm i", " - uses: actions/setup-node@v6\n with: {node-version: '20'}\n - run: npm i", 1), compositeSetup, "incompatible"}, + {"cycle", base, "runs:\n using: composite\n steps:\n - uses: ./.github/actions/install\n", "inconclusive"}, + {"local JS action", base, "runs: {using: node24, main: index.js}", "inconclusive"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := newJestRepository(t, tc.workflow) + writeScripts(t, root, map[string]string{"coverage": "jest", "lint": "eslint ."}) + writeComposite(t, root, "install", tc.action) + result := checkCIRuntimes(t.Context(), root, nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.17.0", Node: ">=22"}, nil + }) + require.Equal(t, tc.status, result.Status, result) + }) + } +} + +func TestCompositeNestingAndPathBoundary(t *testing.T) { + root := t.TempDir() + writeComposite(t, root, "outer", "runs:\n using: composite\n steps:\n - uses: ./.github/actions/inner\n with: {node: '24'}\n") + writeComposite(t, root, "inner", compositeSetup) + steps := expandCompositeSteps(root, "workflow", []runtimeStep{{Uses: "./.github/actions/outer"}}) + require.Len(t, steps, 2) + require.Equal(t, "24", steps[0].With["node-version"]) + require.Equal(t, ".github/actions/inner/action.yml / step 1", steps[0].Source) + out := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(out, "action.yml"), []byte(compositeSetup), 0644)) + require.NoError(t, os.Symlink(out, filepath.Join(root, "outside"))) + _, err := localActionPath(root, "./outside") + require.ErrorContains(t, err, "escapes") + _, err = localActionPath(root, "./../outside") + require.Error(t, err) +} diff --git a/internal/onboard/express_validator_test.go b/internal/onboard/express_validator_test.go new file mode 100644 index 00000000..20b9a6dc --- /dev/null +++ b/internal/onboard/express_validator_test.go @@ -0,0 +1,203 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// Preserve the commands, local composite and matrix guards used in the +// express-validator experiment, including the backticks that execute a script. +const expressValidatorWorkflow = `jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: {node: [14, 16, 18, 20, 22, 24]} + steps: + - uses: actions/setup-node@v3 + with: {node-version: '${{ matrix.node }}'} + - uses: ./.github/actions/npm-cache + - run: npm ci + - run: npm run build + - uses: datadog/test-visibility-github-action@v3 + if: matrix.node >= 22 + with: {languages: js, js-tracer-version: '6.17.0'} + - run: npm test + if: matrix.node >= 22 + env: {NODE_OPTIONS: '-r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}'} + - run: npm test + if: matrix.node < 22 + docs: + steps: + - uses: ./.github/actions/npm-cache + - run: npm run docs:regenerate-api + - run: git diff --quiet --exit-code || echo "Regenerate docs with ` + "`npm run docs:regenerate-api`" + `" + - run: npm run docs:build +` + +func expressValidatorRepository(t *testing.T) string { + t.Helper() + root := newJestRepository(t, expressValidatorWorkflow) + writeScripts(t, root, map[string]string{ + "test": "jest", "build": "tsc", "docs:build": "npm --prefix ./website run build", + "docs:regenerate-api": "npm --prefix ./website run regenerate-api", + }) + writeScripts(t, filepath.Join(root, "website"), map[string]string{ + "build": "docusaurus build", "regenerate-api": "node ./scripts/regenerate-api.js", + }) + action := filepath.Join(root, ".github/actions/npm-cache") + require.NoError(t, os.MkdirAll(action, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(action, "action.yml"), []byte(`runs: + using: composite + steps: + - run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT + shell: bash + - uses: actions/cache@v3 +`), 0644)) + return root +} + +func TestExpressValidatorCIAndCommandDiscovery(t *testing.T) { + root := expressValidatorRepository(t) + command, err := JestValidationCommand(root) + require.NoError(t, err) + require.Equal(t, "npm test", command) + result := checkCIRuntimes(t.Context(), root, nil, func(_ context.Context, action, version string) (tracerRequirement, error) { + require.Equal(t, githubAction+"@v3", action) + require.Equal(t, "6.17.0", version) + return tracerRequirement{Version: version, Node: ">=22"}, nil + }) + require.Equal(t, "compatible", result.Status, result) + require.Len(t, result.Jobs, 6) + for i, finding := range result.Jobs { + expected := "excluded" + if i >= 4 { + expected = "compatible" + } + require.Equal(t, expected, finding.Status, finding) + require.Equal(t, "npm test", finding.Command) + } + require.Len(t, result.Review, 3, "custom docs scripts remain explicit review items") + for _, review := range result.Review { + require.Equal(t, "docs", review.Job) + require.Equal(t, "not checked", review.Status) + } + discovery, err := findWorkflows(root, "javascript", "jest") + require.NoError(t, err) + require.Empty(t, discovery.Unresolved) + require.Len(t, discovery.Review, 3) +} + +func TestUnknownCIEntryDoesNotBlockKnownLocalInvocation(t *testing.T) { + root := newJestRepository(t, "jobs:\n tests:\n steps:\n - run: node unknown.js\n - run: npm test\n") + writeScripts(t, root, map[string]string{"test": "jest"}) + command, err := JestValidationCommand(root) + require.NoError(t, err) + require.Equal(t, "npm test", command) + result := checkCIRuntimes(t.Context(), root, nil, nil) + require.Equal(t, "incompatible", result.Status) // Known Jest step also lacks instrumentation. + require.Contains(t, result.Jobs[0].Reason, "unknown.js") +} + +func TestNpmPrefixFollowsDirectoryAndLifecycle(t *testing.T) { + root := t.TempDir() + child := filepath.Join(root, "website") + writeScripts(t, root, map[string]string{"test": "eslint .", "build": "npm --prefix ./website run build"}) + writeScripts(t, child, map[string]string{"test": "jest", "build": "docusaurus build"}) + for _, command := range []string{"npm --prefix ./website test", "npm --prefix=./website run test", "CI=true npm --prefix website test"} { + result := resolveJestCommand(root, command, nil) + require.True(t, result.Matched, result) + require.False(t, result.SingleJest, "subdirectory validation needs explicit working directory") + require.Empty(t, result.Reason) + } + result := resolveJestCommand(root, "npm run build", nil) + require.Empty(t, result.Reason, "same script name in a different package is not a cycle") + require.Contains(t, result.ReviewReason, "docusaurus") + writeScripts(t, root, map[string]string{"test": "jest", "build": "npm --prefix ./website run build"}) + require.True(t, resolveJestCommand(root, "npm --prefix . test", nil).SingleJest) + writeScripts(t, child, map[string]string{"build": "npm --prefix .. run build"}) + require.Contains(t, resolveJestCommand(root, "npm run build", nil).Reason, "cycle") + for _, hook := range []string{"pretest", "posttest"} { + writeScripts(t, child, map[string]string{"test": "jest", hook: "node setup.js"}) + require.Contains(t, resolveJestCommand(root, "npm --prefix website test", nil).Reason, hook) + } + outside := t.TempDir() + writeScripts(t, outside, map[string]string{"test": "jest"}) + require.NoError(t, os.Symlink(outside, filepath.Join(root, "outside"))) + for _, prefix := range []string{"../outside", "outside", outside, "missing", "$TARGET", "'~'", "''"} { + result := resolveJestCommand(root, "npm --prefix "+prefix+" test", nil) + require.False(t, result.Matched, result) + require.NotEmpty(t, result.Reason, result) + } +} + +func TestMetadataNeverHidesExecutedTestsOrUnknownCommands(t *testing.T) { + root := expressValidatorRepository(t) + for _, command := range []string{ + `echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT`, + `printf '%s\n' "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT"`, + } { + result := resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{Run: command}, "javascript", "jest") + require.Empty(t, result.Reason, result) + require.False(t, result.Matched) + } + for _, command := range []string{ + `echo "$(npm test)" >> $GITHUB_OUTPUT`, + "git diff --quiet --exit-code || echo \"`npm test`\"", + `echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT && npm test`, + } { + result := resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{Run: command}, "javascript", "jest") + require.True(t, result.Matched, result) + require.False(t, result.Review) + require.False(t, result.SingleJest) + } + for _, command := range []string{ + `echo "$(node tests.js)"`, `echo "$(npm config get cache; node tests.js)" >> $GITHUB_OUTPUT`, + `echo "$(npm run missing)"`, `echo "$(npm run docs:build)" >> $GITHUB_ENV`, + `echo "${VALUE:-$(npm test)}"`, `echo "$(npm --prefix "$TARGET" test)"`, + `echo "$(echo $(npm test))"`, `$(npm config get cache)`, `echo okay > $(npm test)`, + `git diff --quiet --exit-code || echo "$UNKNOWN"`, `git diff --quiet --exit-code || node tests.js`, + } { + result := resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{Run: command}, "javascript", "jest") + require.NotEmpty(t, result.Reason, command) + require.False(t, result.Review, command) + } + result := resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{ + Run: "git diff --quiet --exit-code || echo \"`npm run docs:regenerate-api`\"", + Env: map[string]string{"NODE_OPTIONS": "-r dd-trace/ci/init"}, + }, "javascript", "jest") + require.False(t, result.Review, "an instrumented opaque wrapper must remain unresolved") + require.NotEmpty(t, result.Reason) + writeScripts(t, filepath.Join(root, "website"), map[string]string{"regenerate-api": "jest"}) + result = resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{Run: "git diff --quiet --exit-code || echo \"`npm run docs:regenerate-api`\""}, "javascript", "jest") + require.True(t, result.Matched, "a docs alias that runs Jest must be identified") + require.False(t, result.Review) + writeScripts(t, filepath.Join(root, "website"), map[string]string{"regenerate-api": "node docs.js", "preregenerate-api": "jest"}) + result = resolveTestStep(root, ciWorkflow{}, runtimeJob{}, runtimeStep{Run: "npm run docs:regenerate-api"}, "javascript", "jest") + require.Contains(t, result.Reason, "lifecycle") + require.False(t, result.Review) +} + +func TestConditionErrorIdentifiesCurrentAction(t *testing.T) { + root := expressValidatorRepository(t) + path := filepath.Join(root, ".github/workflows/test.yml") + workflow := strings.Replace(expressValidatorWorkflow, "if: matrix.node >= 22", "if: github.event_name == 'push'", 1) + require.NoError(t, os.WriteFile(path, []byte(workflow), 0644)) + result := checkCIRuntimes(t.Context(), root, nil, nil) + require.Equal(t, "inconclusive", result.Status) + for _, finding := range result.Jobs { + require.Equal(t, 5, finding.Step) + require.Equal(t, githubAction+"@v3", finding.Action) + require.Empty(t, finding.Command) + require.Contains(t, finding.Reason, "github.event_name") + } +} diff --git a/internal/onboard/instructions/github.md b/internal/onboard/instructions/github.md index be45402b..760df312 100644 --- a/internal/onboard/instructions/github.md +++ b/internal/onboard/instructions/github.md @@ -2,6 +2,14 @@ Apply this setup to every GitHub Actions job that runs __FRAMEWORK__. +For Jest and supported Vitest projects, do the work in this order: + +1. Inspect the actual test command and run `ddtest testdrive --check-only --yes` to resolve the framework, the local Node runtime and a tracer candidate. This does not install the tracer or run the suite. Use any generated recommendation instead of guessing older tracer majors. +2. Configure CI below, pinning the exact recommended/resolved release in `js-tracer-version` and using that same release with local `--tracer-version`. Review the recommendation's compatible, unsupported and unresolved CI runtimes. Keep unsupported matrix entries running their existing tests without instrumentation; unresolved entries remain unverified. Existing project dependencies take precedence and require an explicit dependency decision, not a silent replacement. +3. Complete workflow, dependency, Jest configuration and Git/package exclusion edits. Run `--check-only --yes` with the selected tracer and test command again. Fix concrete blockers before full validation; preserve and disclose unresolved tool limitations. +4. Run one full `ddtest testdrive --yes` with that same selection. Wait for completion, then clean up and report its generated verdict. Do not finish with another configuration-only check or edit validated inputs afterward; either requires a new full run. + + ## 1. Instrument the test job Add this step after checkout and dependency installation, immediately before the first test step: @@ -11,7 +19,6 @@ Add this step after checkout and dependency installation, immediately before the uses: datadog/test-visibility-github-action@v3 with: languages: __LANGUAGE__ - __TRACER_SETTING__ api_key: ${{ secrets.DD_API_KEY }} site: datadoghq.com ``` @@ -20,17 +27,55 @@ If the organization uses a Datadog site other than US1, replace `datadoghq.com` __BOOTSTRAP__ +Jest CI discovery follows ordinary package.json script aliases, including nested scripts, using the workflow, job, or step working directory and literal npm `--prefix` package directories. It inspects supported cache-output commands and commands executed by substitutions in metadata steps; echoed text alone is not evidence that a command is harmless. Discovery only reads these files; it never executes scripts. Identified Jest steps are checked. Unresolved build, publishing, and documentation entry points are listed separately for review; inspect them if they also run tests. Unknown test wrappers, dynamic commands, unsupported shells, and unresolved working directories remain inconclusive. Identify their actual test steps without rewriting valid commands just to satisfy discovery. + Keep the existing test command and unrelated workflow content unchanged. Add the Datadog action once per test job, not once per test step. ## 2. Try it locally -Run the local, credential-free setup check: +Identify the actual test command from the package scripts and CI, including custom config paths and required setup. Preserve that command in CI. For local Jest validation, testdrive automatically selects a unique, statically resolved root-level CI command. Supported literal && package scripts are expanded with options forwarded only to their single Jest invocation, preserving setup order, environment and Jest options. The executed command is recorded in the report. Recognized ESLint/Prettier commands still run in the full-suite comparison, but are skipped while the synthetic probe exists so its formatting does not block feature checks; those runs record `probe_lint_skipped`. Other setup remains in place. Check workflow environment such as TZ, LANG and test selection variables against the local environment; carry over required literal values consistently to both modes. Do not copy CI secrets or silently reduce the test suite. Unresolved commands elsewhere do not prevent a known invocation from running locally; they remain independent CI findings. If no known or unresolved CI invocation is found, it tries the `test:ci` and `test` scripts before the runner default. Ambiguous commands, lifecycle hooks, and scripts that cannot forward Jest options safely require an explicit `--command` after reviewing setup. An explicit command overrides automatic selection. Do not silently drop a non-default config or necessary setup. + +For Jest, use `ddtest testdrive --check-only --yes` during the setup sequence above (and `--command` when needed). This executes Jest's `--showConfig`, but does not install a tracer or run tests. Review the detected Jest version, runner, Node version, and tracer selection. Fix known incompatibilities before the full run: older Jest may need a supported tracer major and a matching `jest-circus` runner. Avoid upgrading the framework or dropping CI coverage merely to use the newest tracer. + +`--tracer-version ` selects the fallback tracer. An existing project tracer takes precedence, which the report states explicitly. Otherwise ddtest resolves the selector once and installs that exact version in a temporary directory. Use the resolved version in the action's `js-tracer-version` input to validate the same tracer locally and in CI. There is no universal tracer version to copy into every repository. + +After CI configuration checks and all intended edits, run the full local, credential-free validation with the same command and tracer selection. Run it sequentially: wait for testdrive to finish before starting lint, build, or another test run, because feature validation briefly creates a probe test in the repository: ```shell -ddtest testdrive +ddtest testdrive --yes ``` -After it finishes, post every `Open report:` link printed by `ddtest testdrive` to the user so they can open the local Test Optimization report. Do not only summarize the result. +Testdrive reuses an existing project tracer. JavaScript and Python fallback installations are temporary and cleaned up afterward; do not add them to the project dependencies for local validation. Ruby fallback uses `bundle add datadog-ci` and retains its Gemfile and lockfile changes. It retains `.testoptimization/report.html` using the upstream HTML renderer, plus `.testoptimization/testdrive.json`, updating the latest invocation and retaining at most one earlier paired execution under `retained_execution.result`. Later configuration-only, unsupported-framework, or setup-failure runs preserve that evidence; a new paired execution supersedes it. The top-level `last_execution` and `configuration_check` identify the evidence for the local and configuration statuses separately. Retained evidence includes its original timestamp, tracer, commands, counts, and feature verdicts. It is explicitly historical and is never reused to declare the current invocation successful. The compact report records overall success, verdicts, test commands, modes, exit codes, and aggregate counts; it keeps a bounded failure-output excerpt for diagnosis, but no full logs or raw events. Keep both reports after cleanup, even when validation fails. The HTML shows the latest instrumented suite findings; synthetic feature probes do not replace it. Use the JSON for compatibility, feature checks and the overall verdict. Configuration-only checks leave any previous HTML unchanged, so do not present it as a new execution. Do not delete either report. Exclude `.testoptimization/` from source control and published packages; for npm projects, check `files`/`.npmignore` as well as `.gitignore`. Jest validation compares uninstrumented and reporting-only results, then checks features with temporary probe tests. Only the isolated probe commands override Jest coverage thresholds; full-suite comparisons retain the original thresholds, and coverage collection stays enabled for feature validation. The report records this adjustment. Generated Jest coverage is redirected to temporary session storage and cleaned up; existing customer coverage is preserved. Do not remove thresholds from the project configuration. Existing test failures are not automatically validation failures. Frameworks without a validation adapter are explicitly unvalidated. + +For Vitest, use the actual Vitest executable and its CI options with `--command` (for example `--command "pnpm vitest --coverage"`). Run any separate required setup first; do not pass a combined lint/typecheck/test script. Run `--check-only --yes` before editing and again after configuration changes, then finish with full validation. Pin the exact resolved tracer in both `js-tracer-version` and `--tracer-version`; the action default may lag npm. Version support follows the selected dd-trace release, with checks for known incompatibilities and no separate major-version allowlist. The adapter currently handles a single Node project using forks or threads, without Vitest typecheck mode. Multiple projects, browser mode and other pools remain explicitly unvalidated; preserve their configuration. Configuration discovery and live checks must succeed; report specific tool limitations if they cannot. The adapter compares native JSON results, verifies matching telemetry, and runs the six controlled feature scenarios under the original project configuration. Probe files and coverage reports are temporary; keep both final reports. CI initialization for Vitest requires both the action's `--require` preload and `--import` ESM loader. + +For Jest projects, testdrive discovers a probe separately under each uniquely named project and records per-project feature results. A project whose probe cannot be discovered or selected remains unvalidated; do not generalize another project’s passing features to it. Preserve the existing Jest configuration when validation exposes a tracer limitation. + +For Jest, testdrive also checks GitHub Actions Node versions against the tracer selected by each Datadog action, using its explicit version or the default from that action ref. It follows repository-local composite actions, including nested setup steps and literal/default inputs, while preserving their order and conditions. Static numeric matrix comparisons such as `matrix.node >= 22` and string membership such as `contains(fromJSON('["18.x", "20.x", "22.x"]'), matrix.node-version)` are supported. Unknown expressions remain inconclusive. A finding with code `missing_instrumentation` is a concrete configuration error: instrument the named workflow/job/step. It is not an unsupported-syntax limitation. Recheck every Jest entry point, including release workflows; keep intentionally excluded unsupported matrix entries explicit. Recognized other test frameworks and build tools are listed separately from Jest; opaque scripts that may run tests require review. It reads public GitHub and npm metadata; it does not run CI. setup-node LTS aliases (such as `lts/*`) resolve to a major using the public setup-node version manifest, with the source and resolution time recorded. Cached LTS patch versions remain unknown, so patch-specific requirements can remain inconclusive. The aliases `current`, `latest`, and `node` resolve to the newest release for the hosted runner platform and architecture using the public Node distribution index; the selected version, platform, source, and time are recorded. Unknown platforms or unavailable metadata remain inconclusive. Literal JSON packaging transformations and npm publication without known lifecycle hooks are listed for separate review; hooks that may run tests remain unresolved. Other CI entry points listed for review are outside this Jest check. Fix known runtime incompatibilities. An inconclusive result caused by unsupported CI syntax, a dynamic matrix, or unavailable metadata is a tool limitation: preserve the workflow, explain the missing evidence, and leave that check unverified. Do not rewrite valid conditions merely to satisfy the checker, or inspect the binary to discover accepted syntax. Human/agent review may be reported separately; it does not turn an unverified programmatic check into a pass. Use `--check-only` during setup to recheck configuration. Complete workflow, dependency, configuration and ignore-file edits before the final full testdrive. Finish with full validation, not another `--check-only`. The report separates `last_execution` (the most recent paired run, its timestamp and whether it is current) from `configuration_check` (this invocation). A later check-only run preserves the earlier execution as passed or incomplete earlier, but not rerun; it cannot certify changed inputs and leaves onboarding incomplete. Run full validation again after further edits before claiming current compatibility or feature success. Prefer a preflight recommended tracer covering the whole existing CI matrix before excluding runtimes. Preserve existing test coverage: unsupported runtime entries can remain uninstrumented, or the user can choose a compatible runtime/tracer. Do not assume the action will resolve runtime incompatibilities. + +Copy `summary.final_response` from the latest JSON verbatim as the opening of your reply, without a completion preamble. Copy its absolute Markdown report links verbatim, each on its own line; do not shorten them to relative paths or replace them with `file://` URLs. Start with its verdict, with no cleanup preamble; keep the counts, clickable HTML/JSON links and cleanup status. Add workflow changes and your own cleanup checks afterward. Its opening paragraph is also stored in `summary.onboarding_response`. It includes the generated verdict, local status and runtime, CI status and runtime scope (instrumented, excluded and unverified entries), and blockers. Keep this scope in the opening response; a local pass does not mean every CI runtime is instrumented or exercised. Explain workflow changes and any historical execution evidence afterward. Do not replace `INCOMPLETE` with an opening claim that onboarding is complete, even when the edits or local tests succeeded. These local feature checks use a mock backend and do not require a real Datadog API key: do not dismiss a failed local scenario as missing backend connectivity or promise that adding credentials will fix it. A skipping path mismatch is diagnostic evidence to investigate, not permission to change Jest roots or test discovery merely to make validation pass. + +Use the latest report's `summary.facts` for counts, repeated runs, concrete CI blockers and the report path. Do not mix earlier counts into the current verdict or claim that a crash is unrelated to instrumentation when the comparison remains inconclusive. A successful repeat does not erase an earlier unexplained crash. + +Also share local compatibility, each feature result, static CI runtime compatibility, local/CI tracer agreement, and the HTML (`Open report`) and `Results JSON` paths separately. Actual CI execution and backend processing remain not exercised by testdrive. Preserve inconclusive results; do not describe receiving telemetry as proof of compatibility. Do not declare validation complete while any required check is failed or inconclusive. + +## Cleanup after local validation + +Avoid building the project unless its test setup requires it. Before any build or +other command that generates files, inspect existing output paths in command +stdout; do not write before/after inventory files. If you need scratch files, +create a unique directory under the supplied TMPDIR (or the system temporary +directory) and remove that exact directory with a shell trap or finally block. +Keep your caches and manifests there; avoid shared /tmp filenames. After +validation, remove only the outputs newly created by your commands (for +example new dist/ or lib/ files), plus your temporary manifests and caches. Do +not delete pre-existing customer outputs, restore over customer edits, or use a +blanket git clean. ddtest already cleans its own probes, tracer and coverage. +Keep the intended onboarding edits, .testoptimization/report.html and +.testoptimization/testdrive.json. Verify Git and package exclusions for both reports even when validation +is incomplete. Confirm cleanup before the final response; disclose anything that +could not be safely removed. Record a concise cleanup conclusion, not an inventory +of every dependency or telemetry event. ## 3. Ask a human to connect Datadog diff --git a/internal/onboard/instructions/vitest.md b/internal/onboard/instructions/vitest.md new file mode 100644 index 00000000..f60e2db3 --- /dev/null +++ b/internal/onboard/instructions/vitest.md @@ -0,0 +1,49 @@ +# Enable Datadog Test Optimization for Vitest on GitHub Actions + +Keep the existing tests, CI matrix, setup and coverage options. Work in this order: + +1. Review the required configurations and CI consumers printed above, including Release jobs and their build prerequisites. Use `ddtest testdrive --all --check-only --yes` to inspect every configuration. For a focused diagnostic only, select an explicit command: + + ```shell + ddtest testdrive --command "pnpm vitest --coverage" --check-only --yes + ``` + + Preserve the original test options; `vitest bench` is a separate, unvalidated mode and must not be instrumented on the strength of these test results. Preflight resolves the framework configuration, local Node and an exact tracer release without installing it or running tests. If metadata fails, fix registry/sandbox access and retry; do not guess an older version from cached web pages or the action default. +2. Add or update the action before the first Vitest step in every test job. Pin the exact resolved release in `js-tracer-version` and use it with local `--tracer-version`. The action default may lag npm. Existing project dependencies take precedence; changing them requires an explicit dependency decision. + + ```yaml + - name: Configure Datadog Test Optimization + uses: datadog/test-visibility-github-action@v3 + with: + languages: js + api_key: ${{ secrets.DD_API_KEY }} + site: datadoghq.com + js-tracer-version: + ``` + + Confirm the customer's Datadog site before connecting CI; US1 is the default above. Merge **both** loaders into the existing Vitest step, preserving any current Node options: + + ```yaml + env: + NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }} + ``` + + Check the tracer against every affected CI Node version. Prefer the preflight recommended candidate covering the whole existing CI matrix before excluding runtimes. Recommendations use current release metadata and framework prerequisites; they do not change dependencies or prove feature support until the live scenarios pass. Local Node compatibility alone is insufficient. Keep unsupported matrix entries running their original tests without instrumentation and report that scope. Preserve valid CI syntax; unresolved static checks remain unverified even after human review. +3. Finish all intended edits, including excluding `.testoptimization/` from Git and published packages (`files`/`.npmignore`). Rerun `--check-only --yes` with the same `--command` and `--tracer-version `. Fix concrete blockers before the full run. +4. Finish with `ddtest testdrive --all --tracer-version --yes`. Listed preparation runs in separate temporary copies; setup failure blocks that configuration. `--check-only` skips execution. All configurations share one HTML/JSON pair. Use `--command` only for focused diagnostics. Wait for completion before other commands. Further edits require a full rerun; do not finish with check-only or copy reports into separate folders. + +## Validation scope + +Version support follows the selected dd-trace release; ddtest does not impose a separate major-version allowlist. Known tracer incompatibilities are checked before execution. Configuration discovery, paired execution and controlled scenarios must still succeed; a preflight pass alone is not validation. This adapter currently handles one Node project using forks or threads, without Vitest typecheck mode. Multiple projects, browser mode and other pools remain explicitly unvalidated; do not change the customer's configuration merely to pass validation. If ddtest cannot inspect the configuration or read runner results, report that specific tool limitation. + +Testdrive compares baseline and reporting-only outcomes with behavior-changing features disabled. Matching existing test failures are acceptable. It then checks retries, early flake detection, skipping, quarantine, disabled tests and attempt-to-fix using temporary probes under the project configuration. Only probe coverage thresholds are relaxed; full-suite thresholds stay intact. Tests use a local mock backend and require no real Datadog credentials. A local feature failure is not fixed by adding an API key. + +## Cleanup and final response + +Testdrive uses temporary tracer installations, probes and coverage storage; it reuses an existing tracer without changing dependencies. Keep `.testoptimization/report.html` and `.testoptimization/testdrive.json`, including after failures. No raw event archive is needed. Preparation copies are removed automatically. Remove only outputs created by your own commands; preserve pre-existing files and customer edits. Keep scratch files/caches under a unique temporary directory and remove that exact directory afterward. Verify Git and package exclusions and report anything that could not be cleaned up. + +Start your final reply by copying `summary.final_response` from the latest JSON verbatim, including its absolute report links. Then list workflow edits and cleanup. Do not add a preamble such as “Onboarding is done.” INCOMPLETE means onboarding is incomplete even if workflow edits are finished. Preserve failed-check explanations; do not invent a cause or attribute them to missing credentials. Check-only evidence is historical. Unvalidated configurations remain untested. + +## Ask a human to connect Datadog + +After local validation, ask the human to follow the [Datadog API key instructions](https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token), create a key in the selected site, add it as the GitHub secret `DD_API_KEY`, and confirm it is ready without sharing the key itself. Do not create/read the secret or request it in chat. After confirmation, commit and push the workflow change. Actual CI execution and Datadog backend processing remain untested until that workflow runs. diff --git a/internal/onboard/jest_command.go b/internal/onboard/jest_command.go new file mode 100644 index 00000000..d3c6826f --- /dev/null +++ b/internal/onboard/jest_command.go @@ -0,0 +1,119 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "encoding/json" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/kballard/go-shellquote" +) + +func stepDirectory(workflow ciWorkflow, job runtimeJob, step runtimeStep) string { + directory := workflow.Defaults.Run.WorkingDirectory + for _, value := range []string{job.Defaults.Run.WorkingDirectory, step.WorkingDirectory} { + if value != "" { + directory = value + } + } + return directory +} + +// JestValidationCommand selects a single root-level CI command without executing +// code. Package scripts must forward added Jest flags to exactly one runner. +// An empty result leaves repositories without a known script on the default. +func JestValidationCommand(root string) (string, error) { + workflows, err := readWorkflows(root) + if err != nil { + return "", err + } + candidates := map[string]bool{} + var unresolved string + for _, workflow := range workflows { + for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { + job := workflow.Jobs[name] + if strings.TrimSpace(job.If) == "false" { + continue + } + for _, step := range job.Steps { + if strings.TrimSpace(step.If) == "false" { + continue + } + resolution := resolveTestStep(root, workflow, job, step, "javascript", "jest") + if resolution.Reason != "" && !resolution.Review { + unresolved = fmt.Sprintf("CI command %q requires review: %s", step.Run, resolution.Reason) + } + if !resolution.Matched { + continue + } + words, err := shellquote.Split(step.Run) + if err != nil || resolution.Reason != "" || len(words) == 0 || filepath.Clean(stepDirectory(workflow, job, step)) != "." { + return "", fmt.Errorf("cannot automatically forward Jest options through CI command %q; use --command with its working directory and required setup", step.Run) + } + selected, err := ForwardJestCommand(root, step.Run) + if err != nil { + return "", err + } + candidates[selected] = true + } + } + } + if len(candidates) > 1 { + return "", fmt.Errorf("multiple Jest CI commands found: %s; choose one with --command and validate the others separately", strings.Join(slices.Sorted(maps.Keys(candidates)), "; ")) + } + for command := range candidates { + return command, nil + } + // A known invocation can be validated locally even when other CI entry + // points are unknown. CheckCIRuntimes still reports those independently. + // Without a known CI command, do not guess past an unresolved wrapper. + if unresolved != "" { + return "", fmt.Errorf("%s; use --command after reviewing required setup", unresolved) + } + data, err := os.ReadFile(filepath.Join(root, "package.json")) + if err != nil { + return "", err + } + var manifest struct { + Scripts map[string]string `json:"scripts"` + PackageManager string `json:"packageManager"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return "", err + } + manager, _, _ := strings.Cut(manifest.PackageManager, "@") + if manager == "" { + manager = "npm" + for _, entry := range []struct{ file, manager string }{{"pnpm-lock.yaml", "pnpm"}, {"yarn.lock", "yarn"}} { + if _, err := os.Stat(filepath.Join(root, entry.file)); err == nil { + manager = entry.manager + break + } + } + } + if !slices.Contains([]string{"npm", "yarn", "pnpm", "bun"}, manager) { + return "", fmt.Errorf("unknown package manager %q; select the Jest command with --command", manager) + } + for _, name := range []string{"test:ci", "test"} { + if manifest.Scripts[name] == "" { + continue + } + command := manager + " run " + name + resolution := resolveJestCommand(root, command, nil) + if resolution.Matched && resolution.Reason == "" { + return ForwardJestCommand(root, command) + } + if resolution.Matched || resolution.Reason != "" { + return "", fmt.Errorf("package script %q requires review before forwarding Jest options; use --command with the actual Jest invocation and setup", name) + } + } + return "", nil +} diff --git a/internal/onboard/jest_forwarding.go b/internal/onboard/jest_forwarding.go new file mode 100644 index 00000000..dc05a02e --- /dev/null +++ b/internal/onboard/jest_forwarding.go @@ -0,0 +1,170 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/kballard/go-shellquote" +) + +// ForwardJestCommand expands only statically understood root package scripts. +// Additional options go to the single Jest invocation, never a lint/setup step. +// CI files are untouched, and && order, environment and Jest options survive. +func ForwardJestCommand(root, command string) (string, error) { + resolution := resolveJestCommand(root, command, nil) + if !resolution.Matched || resolution.Reason != "" { + return command, nil // Explicit custom commands remain the caller's choice. + } + words, err := shellquote.Split(command) + if err != nil { + return "", err + } + if resolution.SingleJest && len(words) > 0 && !strings.Contains(words[0], "=") { + if words[0] == "jest" { + words[0] = filepath.Join(root, "node_modules", ".bin", "jest") + return shellquote.Join(words...), nil + } + return command, nil + } + count := 0 + body, err := forwardJestScript(root, command, 0, &count) + if err != nil || count != 1 { + return "", fmt.Errorf("cannot forward probe options through %q: %v (Jest invocations: %d); use --command with the actual Jest executable, original options and required setup", command, err, count) + } + body = "PATH=" + shellquote.Join(filepath.Join(root, "node_modules", ".bin")) + ":\"$PATH\"; export PATH; " + body + return shellquote.Join("sh", "-c", body, "ddtest-jest"), nil +} + +func forwardJestScript(root, command string, depth int, count *int) (string, error) { + if depth >= 16 || strings.ContainsAny(command, ";\n") { + return "", fmt.Errorf("only bounded literal && sequences can be forwarded") + } + commands, err := staticCommands(command) + if err != nil { + return "", err + } + var parts []string + for _, words := range commands { + for _, word := range words { + if strings.ContainsAny(word, "*?[") { + return "", fmt.Errorf("shell glob expansion requires review") + } + } + prefix := []string{} + for len(words) > 0 && strings.Contains(words[0], "=") { + prefix = append(prefix, words[0]) + words = words[1:] + } + if len(words) == 0 { + return "", fmt.Errorf("standalone environment assignment requires review") + } + part := shellquote.Join(words...) + resolution := resolveJestCommand(root, part, nil) + if resolution.Matched { + switch words[0] { + case "jest", "./node_modules/.bin/jest", "node_modules/.bin/jest": + if slices.Contains(words, "--") { + return "", fmt.Errorf("positional Jest selection requires an explicit command") + } + *count++ + part += ` "$@"` + case "npm", "yarn", "pnpm", "bun": + args := words[1:] + if len(args) > 0 && (args[0] == "run" || args[0] == "run-script") { + args = args[1:] + } + if len(args) == 0 { + return "", fmt.Errorf("missing package script") + } + name := args[0] + if words[0] == "npm" && name == "t" { + name = "test" + } + data, err := os.ReadFile(filepath.Join(root, "package.json")) + if err != nil { + return "", err + } + var manifest struct{ Scripts map[string]string } + if err := json.Unmarshal(data, &manifest); err != nil { + return "", err + } + script, ok := manifest.Scripts[name] + if !ok || manifest.Scripts["pre"+name] != "" || manifest.Scripts["post"+name] != "" { + return "", fmt.Errorf("package script or lifecycle requires review") + } + args = args[1:] + if len(args) > 0 && args[0] == "--" { + args = args[1:] + } + if len(args) > 0 { + script += " " + shellquote.Join(args...) + } + part, err = forwardJestScript(root, script, depth+1, count) + if err != nil { + return "", err + } + default: + return "", fmt.Errorf("wrapper %q requires review", words[0]) + } + } + if !resolution.Matched && lintOnly(root, shellquote.Join(words...), 0) { + part = `if [ "${DDTEST_JEST_PROBE:-0}" = 1 ]; then :; else ` + part + `; fi` + } + if len(prefix) > 0 { + part = shellquote.Join(append([]string{"env"}, prefix...)...) + " " + shellquote.Join("sh", "-c", part, "ddtest-jest") + ` "$@"` + } + parts = append(parts, part) + } + return strings.Join(parts, " && "), nil +} + +// Lint is part of the real suite command, but must not reject a temporary +// synthetic test's formatting before Jest can discover it. Opaque setup stays. +func lintOnly(root, command string, depth int) bool { + if depth >= 16 { + return false + } + commands, err := staticCommands(command) + if err != nil || len(commands) == 0 { + return false + } + for _, words := range commands { + for len(words) > 0 && strings.Contains(words[0], "=") { + words = words[1:] + } + if len(words) == 0 { + return false + } + if words[0] == "eslint" || words[0] == "prettier" { + continue + } + if !slices.Contains([]string{"npm", "yarn", "pnpm", "bun"}, words[0]) { + return false + } + args := words[1:] + if len(args) > 0 && (args[0] == "run" || args[0] == "run-script") { + args = args[1:] + } + if len(args) != 1 { + return false + } + data, err := os.ReadFile(filepath.Join(root, "package.json")) + if err != nil { + return false + } + var manifest struct{ Scripts map[string]string } + if json.Unmarshal(data, &manifest) != nil || manifest.Scripts["pre"+args[0]] != "" || manifest.Scripts["post"+args[0]] != "" || !lintOnly(root, manifest.Scripts[args[0]], depth+1) { + return false + } + } + return true +} diff --git a/internal/onboard/jest_forwarding_test.go b/internal/onboard/jest_forwarding_test.go new file mode 100644 index 00000000..77194fe4 --- /dev/null +++ b/internal/onboard/jest_forwarding_test.go @@ -0,0 +1,115 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/kballard/go-shellquote" + "github.com/stretchr/testify/require" +) + +func TestCompoundJestForwardsOnlyToJestAndPreservesSetup(t *testing.T) { + root := t.TempDir() + writeScripts(t, root, map[string]string{"test": "NODE_ENV=test jest --config unit.js && npm run lint", "lint": "eslint ."}) + bin := filepath.Join(root, "node_modules", ".bin") + require.NoError(t, os.MkdirAll(bin, 0755)) + for name, script := range map[string]string{ + "jest": "#!/bin/sh\nprintf '%s\\n' \"$NODE_ENV:$*\" > jest-args\n", + "npm": "#!/bin/sh\nprintf '%s\\n' \"$*\" > setup-args\n", + } { + require.NoError(t, os.WriteFile(filepath.Join(bin, name), []byte(script), 0755)) + } + selected, err := ForwardJestCommand(root, "npm test") + require.NoError(t, err) + words, err := shellquote.Split(selected) + require.NoError(t, err) + cmd := exec.CommandContext(t.Context(), words[0], append(words[1:], "--listTests", "--json", "--runTestsByPath", "probe file.js")...) + cmd.Dir = root + data, err := cmd.CombinedOutput() + require.NoError(t, err, string(data)) + data, err = os.ReadFile(filepath.Join(root, "jest-args")) + require.NoError(t, err) + require.Equal(t, "test:--config unit.js --listTests --json --runTestsByPath probe file.js\n", string(data)) + data, err = os.ReadFile(filepath.Join(root, "setup-args")) + require.NoError(t, err) + require.Equal(t, "run lint\n", string(data)) +} + +func TestForwardJestRejectsMultipleRunnersAndPreservesUnknownCommands(t *testing.T) { + root := t.TempDir() + writeScripts(t, root, map[string]string{"test": "jest && jest"}) + _, err := ForwardJestCommand(root, "npm test") + require.ErrorContains(t, err, "Jest invocations: 2") + selected, err := ForwardJestCommand(root, "node custom-wrapper.js") + require.NoError(t, err) + require.Equal(t, "node custom-wrapper.js", selected) +} + +func TestSetupFormsAndMissingReleaseInstrumentation(t *testing.T) { + root := newJestRepository(t, `jobs: + tests: + steps: + - uses: actions/setup-node@v4 + with: {node-version: '22'} + - run: npm -g i yarn + - run: yarn + - uses: datadog/test-visibility-github-action@v3 + with: {languages: js} + - run: yarn test + env: {NODE_OPTIONS: '-r ${{ env.DD_TRACE_PACKAGE }}'} + - run: npm run site + release: + steps: + - run: yarn test +`) + writeScripts(t, root, map[string]string{"test": "jest", "site": "documentation build src/index.js"}) + result := checkCIRuntimes(t.Context(), root, nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.17.0", Node: ">=22"}, nil + }) + require.Equal(t, "incompatible", result.Status) + require.Len(t, result.Jobs, 2) + require.Len(t, result.Review, 1) + for _, job := range result.Jobs { + if job.Job == "release" { + require.Equal(t, "missing_instrumentation", job.Code) + require.Contains(t, job.Reason, "release before step") + } else { + require.Equal(t, "compatible", job.Status) + } + } +} + +func TestProbeSkipsOnlyRecognizedLintAndPreservesFullSuiteFailure(t *testing.T) { + root := t.TempDir() + writeScripts(t, root, map[string]string{"test": "yarn lint && jest", "lint": "eslint ./src"}) + bin := filepath.Join(root, "node_modules", ".bin") + require.NoError(t, os.MkdirAll(bin, 0755)) + // A failing linter models the real TSyringe probe's prettier errors. + require.NoError(t, os.WriteFile(filepath.Join(bin, "yarn"), []byte("#!/bin/sh\nexit 7\n"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(bin, "jest"), []byte("#!/bin/sh\nprintf '%s' \"$*\"\n"), 0755)) + selected, err := ForwardJestCommand(root, "yarn test") + require.NoError(t, err) + words, err := shellquote.Split(selected) + require.NoError(t, err) + for _, probe := range []string{"0", "1"} { + cmd := exec.CommandContext(t.Context(), words[0], append(words[1:], "--runTestsByPath", "probe.test.ts")...) + cmd.Dir, cmd.Env = root, append(os.Environ(), "DDTEST_JEST_PROBE="+probe) + output, err := cmd.CombinedOutput() + if probe == "0" { + require.Error(t, err, "real suite must retain lint failure") + } else { + require.NoError(t, err, string(output)) + require.Contains(t, string(output), "--runTestsByPath probe.test.ts") + } + } + writeScripts(t, root, map[string]string{"lint": "node setup.js"}) + require.False(t, lintOnly(root, "yarn lint", 0), "a script name is not proof of lint-only setup") +} diff --git a/internal/onboard/multiframework_test.go b/internal/onboard/multiframework_test.go index fa96d902..32913385 100644 --- a/internal/onboard/multiframework_test.go +++ b/internal/onboard/multiframework_test.go @@ -40,7 +40,6 @@ func TestOnboardAllSupportedFrameworks(t *testing.T) { t.Chdir(root) require.NoError(t, Run(&output)) require.Contains(t, output.String(), "languages: "+language) - require.Contains(t, output.String(), language+"-tracer-version: '' # Latest release") require.Contains(t, output.String(), "ddtest testdrive --framework "+name) require.Contains(t, output.String(), "Ask a human to connect Datadog") if language != "js" { diff --git a/internal/onboard/npm_prefix.go b/internal/onboard/npm_prefix.go new file mode 100644 index 00000000..be9fa19d --- /dev/null +++ b/internal/onboard/npm_prefix.go @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "fmt" + "path/filepath" + "strings" +) + +// Only consume a literal global prefix before the npm subcommand. Other npm +// options remain unresolved instead of being mistaken for forwarded Jest flags. +func npmPrefix(args []string) (string, []string, error) { + if len(args) == 0 { + return "", args, nil + } + var prefix string + if args[0] == "--prefix" { + if len(args) < 3 { + return "", nil, fmt.Errorf("npm --prefix requires a directory and subcommand") + } + prefix, args = args[1], args[2:] + } else if value, ok := strings.CutPrefix(args[0], "--prefix="); ok { + prefix, args = value, args[1:] + } else { + return "", args, nil + } + if prefix == "" || len(args) == 0 || strings.HasPrefix(args[0], "-") { + return "", nil, fmt.Errorf("unsupported npm --prefix selection requires review") + } + return prefix, args, nil +} + +func repositoryDirectory(root, directory, prefix string) (string, error) { + if filepath.IsAbs(prefix) || strings.ContainsAny(prefix, "$`~*?[]{}") { + return "", fmt.Errorf("npm --prefix must name a static repository directory: %s", prefix) + } + target, err := filepath.EvalSymlinks(filepath.Join(directory, prefix)) + if err != nil { + return "", fmt.Errorf("cannot resolve npm --prefix directory: %w", err) + } + boundary, err := filepath.EvalSymlinks(root) + if err != nil { + return "", err + } + relative, err := filepath.Rel(boundary, target) + if err != nil || !filepath.IsLocal(relative) { + return "", fmt.Errorf("npm --prefix directory escapes the repository: %s", prefix) + } + return target, nil +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index 1bd97176..19220e4e 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -10,9 +10,10 @@ import ( _ "embed" "fmt" "io" + "maps" "os" "path/filepath" - "sort" + "slices" "strings" "github.com/DataDog/ddtest/internal/platform" @@ -24,6 +25,9 @@ const githubAction = "datadog/test-visibility-github-action" //go:embed instructions/github.md var gitHubInstructions string +//go:embed instructions/vitest.md +var vitestInstructions string + // Run detects the first supported onboarding path and prints its instructions. func Run(output io.Writer) error { repositoryRoot, err := os.Getwd() @@ -41,7 +45,8 @@ func Run(output io.Writer) error { language := detectedPlatform.Name() name := runner.Name() - workflows, configured, err := findWorkflows(repositoryRoot, language, name) + discovery, err := findWorkflows(repositoryRoot, language, name) + workflows, configured := discovery.Workflows, discovery.Configured if err != nil { return err } @@ -55,13 +60,46 @@ func Run(output io.Writer) error { for _, workflow := range workflows { _, _ = fmt.Fprintf(output, " - %s\n", workflow) } + if name == "jest" || name == "vitest" { + scope, err := DiscoverValidationScope(repositoryRoot, name) + if err != nil { + return err + } + WriteValidationScope(output, scope) + _, _ = fmt.Fprintf(output, "Run `ddtest testdrive --framework %s --all --check-only --yes`, then finish edits and review separate setup, and run `ddtest testdrive --framework %s --all --tracer-version --yes`. This aggregates every discovered configuration in one report pair. A --command run validates only its selected configuration.\n", name, name) + } + if name == "jest" { + command, err := JestValidationCommand(repositoryRoot) + if err != nil { + _, _ = fmt.Fprintf(output, "\nLocal Jest command needs review: %s\n", err) + } else if command != "" { + _, _ = fmt.Fprintf(output, "\nLocal validation command: %s\nTestdrive selects this command automatically; preserve its config, coverage, and execution options. --command overrides this selection.\n", command) + } + } - if len(configured) == len(workflows) { + if len(discovery.Review) > 0 { + _, _ = fmt.Fprintln(output, "\nOther CI entry points to review separately:") + for _, reason := range discovery.Review { + _, _ = fmt.Fprintf(output, " - %s\n", reason) + } + } + if len(discovery.Unresolved) > 0 { + _, _ = fmt.Fprintln(output, "\nCI command discovery is inconclusive:") + for _, reason := range discovery.Unresolved { + _, _ = fmt.Fprintf(output, " - %s\n", reason) + } + _, _ = fmt.Fprintln(output, "Identify the actual test steps before editing CI. Preserve valid commands; do not rewrite them merely to satisfy discovery. Manual review does not turn an unverified programmatic check into a pass.") + } + + if len(discovery.Unresolved) == 0 && len(configured) == len(workflows) { _, _ = fmt.Fprintln(output) _, _ = fmt.Fprintln(output, "Datadog Test Optimization already appears in every detected test workflow.") - _, _ = fmt.Fprintf(output, "Run `ddtest testdrive --framework %s` to check the setup locally.\n", name) - _, _ = fmt.Fprintln(output, "After it finishes, post every `Open report:` link to the user so they can open the local Test Optimization report.") - return nil + _, _ = fmt.Fprintf(output, "This is configuration detection, not completed onboarding. Run `ddtest testdrive --framework %s` to validate locally. Jest and supported Vitest projects also check CI runtime compatibility; configurations without a validation adapter remain unvalidated.\n", name) + _, _ = fmt.Fprintln(output, "After it finishes, share the validation verdict, CI runtime compatibility and feature results using the final response block printed by testdrive. Copy its absolute Markdown report links verbatim, each on its own line; do not shorten them to relative paths or replace them with file:// URLs.") + _, _ = fmt.Fprintln(output, "Keep .testoptimization/report.html and .testoptimization/testdrive.json after cleanup, even on failure. Do not declare validation complete while any required check is failed or inconclusive.") + if name != "vitest" { + return nil + } } _, _ = fmt.Fprintln(output) @@ -69,16 +107,35 @@ func Run(output io.Writer) error { return nil } -func findWorkflows(repositoryRoot, language, name string) ([]string, []string, error) { - var workflows []string - var configured []string - directory := filepath.Join(repositoryRoot, ".github", "workflows") +type workflowDiscovery struct { + Workflows, Configured, Unresolved, Review []string +} + +type runDefaults struct { + WorkingDirectory string `yaml:"working-directory"` + Shell string `yaml:"shell"` +} + +type ciDefaults struct { + Run runDefaults `yaml:"run"` +} + +type ciWorkflow struct { + Path string + Defaults ciDefaults `yaml:"defaults"` + Env map[string]string `yaml:"env"` + Jobs map[string]runtimeJob `yaml:"jobs"` +} + +func readWorkflows(root string) ([]ciWorkflow, error) { + var workflows []ciWorkflow + directory := filepath.Join(root, ".github", "workflows") entries, err := os.ReadDir(directory) if os.IsNotExist(err) { - return workflows, configured, nil + return nil, nil } if err != nil { - return nil, nil, fmt.Errorf("find GitHub Actions workflows: %w", err) + return nil, fmt.Errorf("find GitHub Actions workflows: %w", err) } for _, entry := range entries { if entry.IsDir() || (filepath.Ext(entry.Name()) != ".yml" && filepath.Ext(entry.Name()) != ".yaml") { @@ -87,62 +144,58 @@ func findWorkflows(repositoryRoot, language, name string) ([]string, []string, e path := filepath.Join(directory, entry.Name()) contents, err := os.ReadFile(path) if err != nil { - return nil, nil, fmt.Errorf("read %s: %w", path, err) + return nil, fmt.Errorf("read %s: %w", path, err) } - text := strings.ToLower(string(contents)) - if !looksLikeTestWorkflow(text, language, name) { - continue + var workflow ciWorkflow + if err := yaml.Unmarshal(contents, &workflow); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) } - - relativePath, err := filepath.Rel(repositoryRoot, path) - if err != nil { - return nil, nil, fmt.Errorf("make workflow path relative: %w", err) - } - relativePath = filepath.ToSlash(relativePath) - workflows = append(workflows, relativePath) - isConfigured, err := allTestJobsConfigured(contents, language, name) - if err != nil { - return nil, nil, fmt.Errorf("parse %s: %w", path, err) - } - if isConfigured { - configured = append(configured, relativePath) + workflow.Path = ".github/workflows/" + entry.Name() + for name, job := range workflow.Jobs { + job.Steps = expandCompositeSteps(root, workflow.Path, job.Steps) + workflow.Jobs[name] = job } + workflows = append(workflows, workflow) } - - sort.Strings(workflows) - sort.Strings(configured) - return workflows, configured, nil + return workflows, nil } -func allTestJobsConfigured(contents []byte, language, name string) (bool, error) { - var workflow struct { - Jobs map[string]struct { - Steps []struct { - Run string `yaml:"run"` - Uses string `yaml:"uses"` - } `yaml:"steps"` - } `yaml:"jobs"` - } - if err := yaml.Unmarshal(contents, &workflow); err != nil { - return false, err - } - foundTestJob := false - for _, job := range workflow.Jobs { - var commands []string - configured := false - for _, step := range job.Steps { - commands = append(commands, strings.ToLower(step.Run)) - configured = configured || strings.Contains(strings.ToLower(step.Uses), githubAction) - } - if !looksLikeTestJob(strings.Join(commands, "\n"), language, name) { - continue +func findWorkflows(root, language, name string) (workflowDiscovery, error) { + var result workflowDiscovery + workflows, err := readWorkflows(root) + if err != nil { + return result, err + } + for _, workflow := range workflows { + found, configured := false, true + for _, jobName := range slices.Sorted(maps.Keys(workflow.Jobs)) { + job := workflow.Jobs[jobName] + hasAction := false + for i, step := range job.Steps { + uses, _, _ := strings.Cut(strings.ToLower(step.Uses), "@") + hasAction = hasAction || uses == githubAction + resolution := resolveTestStep(root, workflow, job, step, language, name) + if resolution.Review { + result.Review = append(result.Review, fmt.Sprintf("%s / %s / step %d (%s): %s", workflow.Path, jobName, stepNumber(step, i), step.Run, resolution.Reason)) + continue + } + if resolution.Matched || resolution.Reason != "" { + found = true + configured = configured && hasAction && resolution.Reason == "" + } + if resolution.Reason != "" { + result.Unresolved = append(result.Unresolved, fmt.Sprintf("%s / %s / step %d (%s): %s", workflow.Path, jobName, stepNumber(step, i), step.Run, resolution.Reason)) + } + } } - foundTestJob = true - if !configured { - return false, nil + if found { + result.Workflows = append(result.Workflows, workflow.Path) + if configured { + result.Configured = append(result.Configured, workflow.Path) + } } } - return foundTestJob, nil + return result, nil } func looksLikeTestJob(commands, language, name string) bool { @@ -163,25 +216,10 @@ func looksLikeTestJob(commands, language, name string) bool { return false } -func looksLikeTestWorkflow(workflow, language, name string) bool { - markers := []string{name, githubAction} - switch language { - case "javascript": - markers = append(markers, "npm test", "npm run test", "yarn test", "yarn run test", "pnpm test", "pnpm run test", "bun test", "bun run test") - case "ruby": - markers = append(markers, "bundle exec rake", "rake test", "rails test") - case "python": - markers = append(markers, "tox", "nox") - } - for _, marker := range markers { - if strings.Contains(workflow, marker) { - return true - } - } - return false -} - func instructions(language, name string) string { + if language == "javascript" && name == "vitest" { + return strings.ReplaceAll(vitestInstructions, "ddtest testdrive", "ddtest testdrive --framework vitest") + } actionLanguage := language var bootstrap string switch language { @@ -199,13 +237,11 @@ func instructions(language, name string) string { case "ruby": bootstrap = rubyBootstrap } - // Empty inputs override the action's pinned defaults and request the latest release. - tracerSetting := actionLanguage + "-tracer-version: '' # Latest release" - text := strings.NewReplacer("__FRAMEWORK__", name, "__LANGUAGE__", actionLanguage, "__BOOTSTRAP__", bootstrap, "__TRACER_SETTING__", tracerSetting).Replace(gitHubInstructions) + text := strings.NewReplacer("__FRAMEWORK__", name, "__LANGUAGE__", actionLanguage, "__BOOTSTRAP__", bootstrap).Replace(gitHubInstructions) return strings.ReplaceAll(text, "ddtest testdrive", "ddtest testdrive --framework "+name) } -const javascriptBootstrap = "Use Node.js 22 or newer. GitHub Actions cannot set NODE_OPTIONS for later steps, so merge this into the existing test step, preserving any current Node options:\n\n```yaml\nenv:\n NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}\n```\n\nThe --import loader is required for Vitest and other ESM tests." +const javascriptBootstrap = "For every instrumented CI matrix entry, use a Node.js version supported by the tracer selected by the action. The Jest and Vitest testdrives check the action's actual tracer metadata; do not infer CI compatibility from the local Node version. GitHub Actions cannot set NODE_OPTIONS for later steps, so merge this into the existing test step, preserving any current Node options:\n\n```yaml\nenv:\n NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}\n```\n\nThe --import loader is required for Vitest and other ESM tests." const pythonBootstrap = "The action exports PYTHONPATH and PYTEST_ADDOPTS=--ddtrace for pytest. Preserve these variables on the existing test step; do not replace its current arguments. Activate the same Python environment used for the tests before the action. If CI uses tox or nox, pass DD_*, PYTHONPATH, and PYTEST_ADDOPTS into the test environment." diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go index e7c41498..41b01155 100644 --- a/internal/onboard/onboard_test.go +++ b/internal/onboard/onboard_test.go @@ -35,7 +35,7 @@ jobs: "api_key: ${{ secrets.DD_API_KEY }}", "NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }}", "ddtest testdrive", - "post every `Open report:` link", + "share local compatibility", "Ask a human to connect Datadog", "without sharing the key itself", } { @@ -70,7 +70,7 @@ jobs: if !strings.Contains(output.String(), "ddtest testdrive") { t.Fatalf("Run() did not print the next step:\n%s", output.String()) } - if !strings.Contains(output.String(), "post every `Open report:` link") { + if !strings.Contains(output.String(), "share the validation verdict") { t.Fatalf("Run() did not tell the agent to share the report:\n%s", output.String()) } } @@ -117,7 +117,7 @@ func TestRunTreatsRepositoryRootAsLiteralPath(t *testing.T) { } func TestRunRequiresGitHubJestWorkflow(t *testing.T) { - repositoryRoot := newJestRepository(t, "name: lint\njobs:\n lint:\n steps:\n - run: npm run lint\n") + repositoryRoot := newJestRepository(t, "name: lint\njobs:\n lint:\n steps:\n - run: npx eslint .\n") t.Chdir(repositoryRoot) err := Run(&bytes.Buffer{}) @@ -144,3 +144,19 @@ func newJestRepository(t *testing.T, workflow string) string { } return repositoryRoot } + +func TestInstructionsUseV3WithoutTracerPinsAndKeepBothReports(t *testing.T) { + for language, framework := range map[string]string{"javascript": "jest", "python": "pytest", "ruby": "rspec"} { + output := instructions(language, framework) + for _, expected := range []string{"datadog/test-visibility-github-action@v3", "Results JSON", "report.html", "testdrive.json", "JavaScript and Python fallback installations are temporary", "Ruby fallback uses `bundle add datadog-ci`", "Fix known runtime incompatibilities", "preserve the workflow", "--check-only", "--tracer-version"} { + if !strings.Contains(output, expected) { + t.Errorf("%s instructions missing %q", language, expected) + } + } + for _, absent := range []string{"-tracer-version:", "__TRACER_SETTING__"} { + if strings.Contains(output, absent) { + t.Errorf("%s instructions still contain %q", language, absent) + } + } + } +} diff --git a/internal/onboard/packaging.go b/internal/onboard/packaging.go new file mode 100644 index 00000000..9123cb62 --- /dev/null +++ b/internal/onboard/packaging.go @@ -0,0 +1,68 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/kballard/go-shellquote" +) + +// Recognize only one literal jq filter with JSON input/output paths. Do not +// relax the general shell parser for pipelines, expansion or chained commands. +var jsonRedirect = regexp.MustCompile("^jq[ \\t]+(?:'[^']*'|\"[^\"$`]*\")[ \\t]+([a-zA-Z0-9_./-]+\\.json)[ \\t]*>[ \\t]*([a-zA-Z0-9_./-]+\\.json)[ \\t]*$") + +func jsonPackagingStep(command string) bool { + match := jsonRedirect.FindStringSubmatch(command) + return len(match) == 3 && filepath.IsLocal(match[1]) && filepath.IsLocal(match[2]) +} + +func resolveNpmPublish(directory string, words []string, framework string) commandResolution { + command := shellquote.Join(words...) + if words[0] == "pnpm" { + words = slices.DeleteFunc(slices.Clone(words), func(word string) bool { return word == "--no-git-checks" }) + } + if len(words) > 3 || (len(words) == 3 && (strings.HasPrefix(words[2], "-") || !filepath.IsLocal(words[2]))) { + return unresolvedCommand("Dynamic or unsupported package publish target requires review: " + shellquote.Join(words...)) + } + targets := []string{directory} + if len(words) == 3 && words[2] != "." { + targets = append(targets, filepath.Join(directory, words[2])) + } + for _, target := range targets { + data, err := os.ReadFile(filepath.Join(target, "package.json")) + if os.IsNotExist(err) { + continue // A generated publication directory is reviewed separately. + } + if err != nil { + return unresolvedCommand("Cannot inspect package publish lifecycle scripts: " + err.Error()) + } + var manifest struct { + Scripts map[string]string `json:"scripts"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return unresolvedCommand("Cannot inspect package publish manifest: " + err.Error()) + } + hooks := []string{"prepublishOnly", "prepack", "prepare", "postpack", "publish", "postpublish"} + if words[0] == "pnpm" { + hooks = append(hooks, "prepublish") + } + for _, hook := range hooks { + if hook == "publish" && strings.TrimSpace(manifest.Scripts[hook]) == "clean-publish" { + continue + } + if manifest.Scripts[hook] != "" { + return unresolvedCommand("Package publish lifecycle script " + hook + " in " + target + " requires review; it may run tests") + } + } + } + return commandResolution{ReviewReason: "Package publication is outside " + framework + " validation; review separately, including lifecycle scripts in generated package manifests: " + command} +} diff --git a/internal/onboard/release.go b/internal/onboard/release.go new file mode 100644 index 00000000..84dc49b4 --- /dev/null +++ b/internal/onboard/release.go @@ -0,0 +1,63 @@ +package onboard + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "slices" + "strings" +) + +// Treat a workflow input as release data only in this complete command form. +// Never discard substitutions or trailing shell commands that might run tests. +var releaseVersionInput = regexp.MustCompile(`^(npm|pnpm|yarn)[ \t]+version[ \t]+\$\{\{[ \t]*(github\.event\.inputs|inputs)\.[a-zA-Z0-9_-]+[ \t]*\}\}[ \t]*$`) + +func resolveReleaseInput(directory, command, framework string) (commandResolution, bool) { + if !releaseVersionInput.MatchString(strings.TrimSpace(command)) { + return commandResolution{}, false + } + result := reviewReleaseLifecycle(directory, command, framework, []string{"preversion", "version", "postversion"}) + if result.Reason == "" { + result.Review, result.Reason = true, result.ReviewReason + } + return result, true +} + +func gitReleaseOperation(words []string) bool { + if len(words) >= 3 && words[1] == "push" { + return true + } + if len(words) < 4 || words[1] != "config" { + return false + } + args := words[2:] + if args[0] == "--global" || args[0] == "--local" { + args = args[1:] + } + return len(args) == 2 && slices.Contains([]string{"user.name", "user.email"}, args[0]) +} + +func reviewReleaseLifecycle(directory, command, framework string, hooks []string) commandResolution { + data, err := os.ReadFile(filepath.Join(directory, "package.json")) + if err != nil { + return unresolvedCommand("Cannot inspect release lifecycle scripts: " + err.Error()) + } + var manifest struct { + Scripts map[string]string `json:"scripts"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return unresolvedCommand("Cannot inspect release lifecycle scripts: " + err.Error()) + } + for _, hook := range hooks { + // A recognized publication wrapper is still reviewed separately; it is + // not an opaque test hook. Other scripts, including chains, stay blocked. + if hook == "publish" && strings.TrimSpace(manifest.Scripts[hook]) == "clean-publish" { + continue + } + if manifest.Scripts[hook] != "" { + return unresolvedCommand("Release lifecycle script " + hook + " requires review alongside " + command + "; it may run tests") + } + } + return commandResolution{ReviewReason: "Release operation is outside " + framework + " validation; review separately, including publication configuration and generated package hooks: " + command} +} diff --git a/internal/onboard/runtime.go b/internal/onboard/runtime.go new file mode 100644 index 00000000..e7296fcc --- /dev/null +++ b/internal/onboard/runtime.go @@ -0,0 +1,378 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "encoding/json" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "slices" + "strings" + "time" + + "go.yaml.in/yaml/v3" +) + +// RuntimeCheck is separate from local test compatibility: a local run cannot +// validate the other runtimes selected by a CI workflow. +type RuntimeCheck struct { + Status string `json:"status"` + Reason string `json:"reason"` + Jobs []RuntimeFinding `json:"jobs,omitempty"` + Review []RuntimeFinding `json:"review,omitempty"` +} + +// RuntimeFinding records the metadata used to check one CI runtime. +type RuntimeFinding struct { + Workflow string `json:"workflow"` + Job string `json:"job"` + Step int `json:"step,omitempty"` + Source string `json:"source,omitempty"` + Command string `json:"command,omitempty"` + Resolution string `json:"resolution,omitempty"` + Node string `json:"node,omitempty"` + NodeSource string `json:"node_source,omitempty"` + NodeResolution *NodeResolution `json:"node_resolution,omitempty"` + Action string `json:"action,omitempty"` + Tracer string `json:"tracer,omitempty"` + TracerRequested string `json:"tracer_requested,omitempty"` + TracerFloating bool `json:"tracer_floating"` + Requirement string `json:"requirement,omitempty"` + Status string `json:"status"` + Reason string `json:"reason"` + Code string `json:"code,omitempty"` +} + +type runtimeStep struct { + Number int `yaml:"-"` + Source string `yaml:"-"` + Parents []string `yaml:"-"` + ResolutionError string `yaml:"-"` + WorkingDirectory string `yaml:"working-directory"` + Shell string `yaml:"shell"` + Env map[string]string `yaml:"env"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + If string `yaml:"if"` + With map[string]string `yaml:"with"` +} + +type runtimeJob struct { + RunsOn any `yaml:"runs-on"` + Defaults ciDefaults `yaml:"defaults"` + Env map[string]string `yaml:"env"` + If string `yaml:"if"` + Strategy struct { + Matrix any `yaml:"matrix"` + } `yaml:"strategy"` + Steps []runtimeStep `yaml:"steps"` +} + +type tracerRequirement struct{ Version, Node, Requested string } +type requirementResolver func(context.Context, string, string) (tracerRequirement, error) + +// CheckCIRuntimes checks Jest or Vitest jobs in GitHub Actions without executing workflow +// code or contacting Datadog. Unknown expressions are never treated as a pass. +func CheckCIRuntimes(ctx context.Context, root string, frameworks ...string) RuntimeCheck { + client := &http.Client{Timeout: 10 * time.Second} + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + return checkCIRuntimes(ctx, root, setupNodeResolver(client), func(ctx context.Context, action, version string) (tracerRequirement, error) { + return resolveRequirement(ctx, client, action, version) + }, frameworks...) +} + +func checkCIRuntimes(ctx context.Context, root string, resolveNode nodeResolver, resolve requirementResolver, frameworks ...string) RuntimeCheck { + framework := "jest" + if len(frameworks) > 0 { + framework = frameworks[0] + } + result := RuntimeCheck{Status: "not applicable", Reason: "No candidate GitHub Actions " + framework + " test commands found; CI runtime compatibility was not checked."} + workflows, err := readWorkflows(root) + if err != nil { + return RuntimeCheck{Status: "inconclusive", Reason: err.Error()} + } + cache := map[string]tracerRequirement{} + cachedResolve := func(ctx context.Context, action, version string) (tracerRequirement, error) { + key := action + "\n" + version + if value, ok := cache[key]; ok { + return value, nil + } + value, err := resolve(ctx, action, version) + if err == nil { + cache[key] = value + } + return value, err + } + for _, workflow := range workflows { + path := workflow.Path + for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { + job := workflow.Jobs[name] + resolutions := make([]commandResolution, len(job.Steps)) + candidate := false + for i, step := range job.Steps { + resolutions[i] = resolveTestStep(root, workflow, job, step, "javascript", framework) + if resolutions[i].Review { + result.Review = append(result.Review, RuntimeFinding{Workflow: path, Job: name, Step: stepNumber(step, i), Command: step.Run, Status: "not checked", Reason: resolutions[i].Reason}) + resolutions[i] = commandResolution{} + } + candidate = candidate || resolutions[i].Matched || resolutions[i].Reason != "" + } + if !candidate { + continue + } + rows, err := runtimeMatrix(job.Strategy.Matrix) + if err != nil { + result.Jobs = append(result.Jobs, RuntimeFinding{Workflow: path, Job: name, Status: "inconclusive", Reason: err.Error()}) + continue + } + if len(rows) == 0 { + result.Jobs = append(result.Jobs, RuntimeFinding{Workflow: path, Job: name, Status: "inconclusive", Reason: "Static matrix has no entries; no instrumented CI runtime could be checked."}) + continue + } + for _, row := range rows { + rowResolutions := slices.Clone(resolutions) + rowCandidate := false + for i, step := range job.Steps { + resolved, _, resolveErr := resolveVariantStep(workflow, job, step, row) + rowResolutions[i] = resolveTestStep(root, workflow, job, resolved, "javascript", framework) + if rowResolutions[i].Review { + rowResolutions[i] = commandResolution{} + } else if resolveErr != nil { + rowResolutions[i].Reason = resolveErr.Error() + } + rowCandidate = rowCandidate || rowResolutions[i].Matched || rowResolutions[i].Reason != "" + } + if !rowCandidate { + continue + } + result.Jobs = append(result.Jobs, checkRuntimeJob(ctx, workflow, name, job, rowResolutions, row, resolveNode, cachedResolve, framework)...) + } + } + } + if len(result.Jobs) == 0 { + return result + } + result.Status = "compatible" + result.Reason = "Every identified instrumented " + framework + " step satisfies its selected tracer's Node requirement. Other entry points listed for review are outside this check. This does not execute CI or verify the Datadog backend." + checked := false + for _, job := range result.Jobs { + checked = checked || job.Status == "compatible" + if job.Status == "incompatible" { + result.Status = "incompatible" + break + } + if job.Status == "inconclusive" { + result.Status = "inconclusive" + } + } + if !checked && result.Status == "compatible" { + result.Status = "inconclusive" + } + if result.Status != "compatible" { + result.Reason = "CI runtime validation is incomplete or incompatible. Fix known incompatibilities. Unknown syntax or metadata requires review; preserve the workflow and report that tool limitation without claiming CI was validated." + } + return result +} + +func checkRuntimeJob(ctx context.Context, workflow ciWorkflow, name string, job runtimeJob, resolutions []commandResolution, row map[string]any, resolveNode nodeResolver, resolve requirementResolver, frameworks ...string) []RuntimeFinding { + finding := RuntimeFinding{Workflow: workflow.Path, Job: name, Status: "inconclusive"} + var findings []RuntimeFinding + fail := func(reason string) []RuntimeFinding { + finding.Status = "inconclusive" + finding.Reason = reason + return append(findings, finding) + } + active, err := runtimeCondition(job.If, nil) + if err != nil { + return fail(err.Error()) + } + if !active { + finding.Status = "excluded" + finding.Reason = "Job is explicitly excluded." + return []RuntimeFinding{finding} + } + var node, nodeSource, nodePlatform, action, version string + skippedAction := false + for i, step := range job.Steps { + uses, _, _ := strings.Cut(strings.ToLower(step.Uses), "@") + resolution := resolutions[i] + test := resolution.Matched || resolution.Reason != "" + if uses != "actions/setup-node" && uses != githubAction && !test { + continue + } + finding = RuntimeFinding{Workflow: workflow.Path, Job: name, Step: stepNumber(step, i), Source: step.Source, Command: step.Run, Resolution: resolution.Evidence, Node: node, NodeSource: nodeSource, Status: "inconclusive"} + if uses == githubAction { + finding.Action = step.Uses + } + active, err := stepCondition(step, row) + if err != nil { + return fail(err.Error()) + } + if !active { + if uses == githubAction { + skippedAction = true + } + continue + } + switch uses { + case "actions/setup-node": + node, err = runtimeValue(step.With["node-version"], row) + nodePlatform = nodeDistributionPlatform(job.RunsOn, step.With["architecture"], row) + nodeSource = step.Source + if err != nil { + return fail(err.Error()) + } + case githubAction: + languages, err := runtimeValue(step.With["languages"], row) + if err != nil { + return fail(err.Error()) + } + if languages != "all" && !slices.Contains(strings.Fields(languages), "js") { + continue + } + action = step.Uses + version, err = runtimeValue(step.With["js-tracer-version"], row) + if err != nil { + return fail(err.Error()) + } + if _, explicit := step.With["js-tracer-version"]; explicit && version == "" { + // An explicit empty input overrides the action default; its + // installer requests npm's latest version instead. + version = "latest" + } + } + if !test { + continue + } + if resolution.Reason != "" { + finding.Reason = "Could not resolve CI test command: " + resolution.Reason + findings = append(findings, finding) + // Unknown siblings do not erase a known test's installation check. + if !resolution.Matched { + continue + } + } + if action == "" { + finding.Status = "incompatible" + finding.Code = "missing_instrumentation" + if skippedAction { + finding.Status = "excluded" + finding.Code = "excluded_runtime" + } + finding.Reason = "No Datadog JavaScript action runs before this test step for this matrix entry. It is not validated as instrumented." + if !skippedAction { + finding.Reason += fmt.Sprintf(" Add tracer installation and initialization in %s / %s before step %d (%s). This is missing instrumentation, not unsupported syntax.", workflow.Path, name, finding.Step, step.Run) + } + } else { + requirement, err := resolve(ctx, action, version) + if err != nil { + return fail(err.Error()) + } + finding.Action = action + finding.Tracer = "dd-trace@" + requirement.Version + finding.TracerRequested = requirement.Requested + if finding.TracerRequested == "" { + finding.TracerRequested = version + } + finding.TracerFloating = finding.TracerRequested != "" && strings.TrimPrefix(finding.TracerRequested, "v") != requirement.Version + finding.Requirement = requirement.Node + resolvedNode := node + if (strings.HasPrefix(node, "lts/") || latestNodeAlias(node)) && resolveNode != nil { + resolution, err := resolveNode(ctx, node, nodePlatform) + if err != nil { + finding.Reason = "Cannot resolve setup-node alias: " + err.Error() + findings = append(findings, finding) + continue + } + finding.NodeResolution = &resolution + resolvedNode = resolution.Version + } + finding.Status, finding.Reason = CompareNodeRequirement(resolvedNode, requirement.Node) + if finding.Status == "compatible" { + if reason := checkJestBootstrap(workflow, job, step, row, frameworks...); reason != "" { + finding.Status, finding.Reason = "inconclusive", reason + finding.Code = "unverified_initialization" + } + } + } + findings = append(findings, finding) + } + if len(findings) == 0 { + return fail("No active test step could be checked.") + } + return findings +} + +func resolveRequirement(ctx context.Context, client *http.Client, action, version string) (tracerRequirement, error) { + if version == "" { + _, ref, ok := strings.Cut(action, "@") + if !ok || ref == "" { + return tracerRequirement{}, fmt.Errorf("datadog action is missing its ref") + } + data, err := fetchRuntimeMetadata(ctx, client, "https://raw.githubusercontent.com/DataDog/test-visibility-github-action/"+url.PathEscape(ref)+"/action.yml") + if err != nil { + return tracerRequirement{}, err + } + var definition struct { + Inputs map[string]struct { + Default string `yaml:"default"` + } `yaml:"inputs"` + } + if err := yaml.Unmarshal(data, &definition); err != nil { + return tracerRequirement{}, fmt.Errorf("decode action metadata: %w", err) + } + version = definition.Inputs["js-tracer-version"].Default + } + if version == "" || strings.Contains(version, "${{") { + return tracerRequirement{}, fmt.Errorf("could not resolve the action's JavaScript tracer version") + } + data, err := fetchRuntimeMetadata(ctx, client, "https://registry.npmjs.org/dd-trace/"+url.PathEscape(version)) + if err != nil { + return tracerRequirement{}, err + } + var manifest struct { + Version string `json:"version"` + Engines struct { + Node string `json:"node"` + } `json:"engines"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return tracerRequirement{}, fmt.Errorf("decode dd-trace metadata: %w", err) + } + if manifest.Version == "" || manifest.Engines.Node == "" { + return tracerRequirement{}, fmt.Errorf("dd-trace metadata has no resolved version or Node requirement") + } + return tracerRequirement{Version: manifest.Version, Node: manifest.Engines.Node, Requested: version}, nil +} + +func fetchRuntimeMetadata(ctx context.Context, client *http.Client, address string) ([]byte, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) + if err != nil { + return nil, err + } + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("read CI runtime metadata: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("read CI runtime metadata from %s: HTTP %d", address, response.StatusCode) + } + const limit = 1 << 20 + data, err := io.ReadAll(io.LimitReader(response.Body, limit+1)) + if err != nil { + return nil, err + } + if len(data) > limit { + return nil, fmt.Errorf("CI runtime metadata exceeds 1 MiB") + } + return data, nil +} diff --git a/internal/onboard/runtime_bootstrap.go b/internal/onboard/runtime_bootstrap.go new file mode 100644 index 00000000..dcc4fde4 --- /dev/null +++ b/internal/onboard/runtime_bootstrap.go @@ -0,0 +1,144 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "fmt" + "strconv" + "strings" + + "github.com/kballard/go-shellquote" +) + +// NUL cannot occur in an environment value. Reject it in source expressions and +// matrix values so only a known action reference can produce these symbols. +const actionPreload = "\x00DD_ACTION_PRELOAD\x00" + +func symbolicConditionValue(value any) bool { + text, ok := value.(string) + return ok && strings.ContainsRune(text, 0) +} + +// Called only after an active JavaScript action. Resolve its known references, +// never arbitrary environment values or commands. Scope overrides need review. +func checkJestBootstrap(workflow ciWorkflow, job runtimeJob, step runtimeStep, row map[string]any, frameworks ...string) string { + known := map[string]string{"DD_TRACE_PACKAGE": actionPreload, "DD_TRACE_ESM_IMPORT": "\x00DD_ACTION_IMPORT\x00"} + var options string + for _, env := range []map[string]string{workflow.Env, job.Env, step.Env} { + if value, ok := env["NODE_OPTIONS"]; ok { + options = value + } + for key := range env { + delete(known, key) + } + } + options, err := bootstrapOptions(options, row, known) + if err == nil && !strings.ContainsAny(options, "$`") { + words, err := shellquote.Split(options) + if err == nil { + requireFound, importFound := false, false + for i, word := range words { + if word == "--require="+actionPreload || ((word == "-r" || word == "--require") && i+1 < len(words) && words[i+1] == actionPreload) { + requireFound = true + } + if word == "--import=\x00DD_ACTION_IMPORT\x00" || (word == "--import" && i+1 < len(words) && words[i+1] == "\x00DD_ACTION_IMPORT\x00") { + importFound = true + } + } + if requireFound && (len(frameworks) == 0 || frameworks[0] != "vitest" || importFound) { + return "" + } + } + } + return "Cannot verify the action's JavaScript preload in this test step's effective NODE_OPTIONS. Preserve custom initialization and report it for review." +} + +func bootstrapOptions(value string, row map[string]any, env map[string]string) (string, error) { + if len(value) > 4096 || strings.ContainsRune(value, 0) { + return "", fmt.Errorf("unsupported NODE_OPTIONS length or NUL byte") + } + var result strings.Builder + for { + literal, expression, found := strings.Cut(value, "${{") + result.WriteString(literal) + if !found { + break + } + p := conditionParser{rest: expression, row: row, env: env} + resolved, err := p.expression(0) + if err != nil || !p.take("}}") { + return "", fmt.Errorf("unsupported NODE_OPTIONS expression") + } + text, ok := resolved.(string) + if !ok { + return "", fmt.Errorf("NODE_OPTIONS expression must resolve to a string") + } + result.WriteString(text) + if result.Len() > 4096 { + return "", fmt.Errorf("resolved NODE_OPTIONS exceeds 4096 characters") + } + value = p.rest + } + if result.Len() > 4096 { + return "", fmt.Errorf("resolved NODE_OPTIONS exceeds 4096 characters") + } + return result.String(), nil +} + +// GitHub format() replaces numbered placeholders and escapes doubled braces. +// Only string arguments are needed for bootstrap paths; other types and format +// specifiers stay inconclusive. Replacement values are never parsed again. +func (p *conditionParser) format() (any, error) { + if !p.take("(") { + return nil, fmt.Errorf("missing format arguments") + } + var args []string + for { + value, err := p.expression(0) + text, ok := value.(string) + if err != nil || !ok { + return nil, fmt.Errorf("unsupported format argument") + } + args = append(args, text) + if !p.take(",") { + break + } + } + if !p.take(")") { + return nil, fmt.Errorf("unclosed format function") + } + var result strings.Builder + for template := args[0]; template != ""; { + switch template[0] { + case '{', '}': + if len(template) > 1 && template[0] == template[1] { + result.WriteByte(template[0]) + template = template[2:] + break + } + i := 1 + for i < len(template) && template[i] >= '0' && template[i] <= '9' { + i++ + } + if template[0] != '{' || i == 1 || i >= len(template) || template[i] != '}' { + return nil, fmt.Errorf("unsupported format placeholder") + } + index, err := strconv.ParseUint(template[1:i], 10, 8) + if err != nil || int(index)+1 >= len(args) { + return nil, fmt.Errorf("format argument index out of range") + } + result.WriteString(args[index+1]) + template = template[i+1:] + default: + result.WriteByte(template[0]) + template = template[1:] + } + if result.Len() > 4096 { + return nil, fmt.Errorf("format result exceeds 4096 characters") + } + } + return result.String(), nil +} diff --git a/internal/onboard/runtime_bootstrap_test.go b/internal/onboard/runtime_bootstrap_test.go new file mode 100644 index 00000000..cce5df62 --- /dev/null +++ b/internal/onboard/runtime_bootstrap_test.go @@ -0,0 +1,125 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const conditionalBootstrap = "${{ matrix.node >= 18 && format('-r {0} --import {1}', env.DD_TRACE_PACKAGE, env.DD_TRACE_ESM_IMPORT) || '' }}" + +func TestConditionalJestBootstrapPreservesMatrixCoverage(t *testing.T) { + for _, guard := range []string{"matrix.node >= 18", "matrix.node >= 22"} { + workflow := fmt.Sprintf(`jobs: + test: + strategy: + matrix: {node: [14, 16, 18, 20, 22, 24]} + steps: + - uses: actions/setup-node@v3 + with: {node-version: '${{ matrix.node }}'} + - uses: datadog/test-visibility-github-action@v3 + if: matrix.node >= 18 + with: {languages: js, js-tracer-version: '5.128.0'} + - run: npm test + env: + NODE_OPTIONS: ${{ %s && format('-r {0} --import {1}', env.DD_TRACE_PACKAGE, env.DD_TRACE_ESM_IMPORT) || '' }} +`, guard) + result := checkCIRuntimes(t.Context(), newJestRepository(t, workflow), nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "5.128.0", Node: ">=18"}, nil + }) + require.Len(t, result.Jobs, 6) + for i, finding := range result.Jobs { + want := "compatible" + switch { + case i < 2: + want = "excluded" + case i < 4 && guard == "matrix.node >= 22": + want = "inconclusive" + } + require.Equal(t, want, finding.Status, finding) + } + } +} + +func TestBootstrapExpressionsAndScope(t *testing.T) { + for _, tc := range []struct { + options string + valid bool + }{ + {conditionalBootstrap, true}, + {"${{ env.DD_TRACE_PACKAGE != '' && format('-r {0}', env.DD_TRACE_PACKAGE) || '' }}", true}, + {"--max-old-space-size=4096 ${{format('--require={0}', env.DD_TRACE_PACKAGE)}}", true}, + {"${{ matrix.node < 18 && '' || format('--require {0}', env.DD_TRACE_PACKAGE) }}", true}, + {"${{format('--require=" + `"{0}"` + "', env.DD_TRACE_PACKAGE)}}", true}, + {"${{format('-r {0}', format('{0}', env.DD_TRACE_PACKAGE))}}", true}, + {"${{ matrix.node < 18 && format('-r {0}', env.DD_TRACE_PACKAGE) || '' }}", false}, + {"${{format('--import {0}', env.DD_TRACE_ESM_IMPORT)}}", false}, + {"${{format('-r {0}', '__DD_ACTION_PRELOAD__')}}", false}, + {"-r " + actionPreload, false}, + {"${{format('-r {0}', matrix.spoof)}}", false}, + {"${{format('-r {0}', env.CUSTOM_PRELOAD)}}", false}, + {"${{ github.event_name == 'push' && format('-r {0}', env.DD_TRACE_PACKAGE) || '' }}", false}, + {"${{ true || env.UNKNOWN }} -r ${{env.DD_TRACE_PACKAGE}}", false}, + {"${{ env.DD_TRACE_PACKAGE != '/custom/path' && format('-r {0}', env.DD_TRACE_PACKAGE) || '' }}", false}, + {"${{ env.DD_TRACE_PACKAGE < '/custom/path' && format('-r {0}', env.DD_TRACE_PACKAGE) || '' }}", false}, + {"${{ !startsWith(env.DD_TRACE_PACKAGE, '/custom') && format('-r {0}', env.DD_TRACE_PACKAGE) || '' }}", false}, + {"${{join(env.DD_TRACE_PACKAGE)}}", false}, + {"${{format('-r {1}', env.DD_TRACE_PACKAGE)}}", false}, + {"${{format('-r {0}', env.DD_TRACE_PACKAGE)}", false}, + {"-r $DD_TRACE_PACKAGE", false}, + {"-r ${{env.DD_TRACE_PACKAGE}} `custom`", false}, + {strings.Repeat(" ", 4097) + "-r ${{env.DD_TRACE_PACKAGE}}", false}, + } { + t.Run(tc.options, func(t *testing.T) { + step := runtimeStep{Env: map[string]string{"NODE_OPTIONS": tc.options}} + reason := checkJestBootstrap(ciWorkflow{}, runtimeJob{}, step, map[string]any{"node": 22, "spoof": actionPreload}) + require.Equal(t, tc.valid, reason == "", reason) + }) + } + for _, scope := range []string{"workflow", "job", "step"} { + workflow, job := ciWorkflow{}, runtimeJob{} + step := runtimeStep{Env: map[string]string{"NODE_OPTIONS": conditionalBootstrap}} + override := map[string]string{"DD_TRACE_PACKAGE": "custom.js"} + switch scope { + case "workflow": + workflow.Env = override + case "job": + job.Env = override + default: + step.Env["DD_TRACE_PACKAGE"] = "custom.js" + } + require.NotEmpty(t, checkJestBootstrap(workflow, job, step, map[string]any{"node": 22}), scope) + } +} + +func TestBootstrapFormatEscapesAndBounds(t *testing.T) { + for _, tc := range []struct{ expression, want string }{ + {"${{format('{{{0}}}', 'value')}}", "{value}"}, + {"${{format('}} {0} {{', 'it''s')}} trailing", "} it's { trailing"}, + {"${{format('{0}', '{1}')}}", "{1}"}, + {"${{format('{01}/{0}/{1}', 'a', 'b')}}", "b/a/b"}, + {"${{format('literal')}}", "literal"}, + } { + got, err := bootstrapOptions(tc.expression, nil, nil) + require.NoError(t, err, tc.expression) + require.Equal(t, tc.want, got) + } + for _, expression := range []string{ + "format()", "format('{')", "format('}')", "format('{0}')", "format('{256}', 'x')", + "format('{-1}', 'x')", "format('{0:foo}', 'x')", "format('{0}', 42)", + "format('{0}{0}', matrix.large)", strings.Repeat("format('{0}', ", 33) + "'x'" + strings.Repeat(")", 33), + } { + _, err := bootstrapOptions("${{"+expression+"}}", map[string]any{"large": strings.Repeat("x", 3000)}, nil) + require.Error(t, err, expression) + } + _, err := bootstrapOptions("${{ matrix.large }}"+strings.Repeat("x", 2000), map[string]any{"large": strings.Repeat("x", 3000)}, nil) + require.Error(t, err) +} diff --git a/internal/onboard/runtime_conditions.go b/internal/onboard/runtime_conditions.go new file mode 100644 index 00000000..e52305fb --- /dev/null +++ b/internal/onboard/runtime_conditions.go @@ -0,0 +1,353 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "unicode" +) + +// A bounded subset of GitHub expressions, not a workflow interpreter. Unknown +// contexts/functions stay inconclusive, even inside short-circuited expressions. +// Scalar types matter: the string "false" is truthy; boolean false is not. +func runtimeCondition(value string, row map[string]any) (bool, error) { + value = strings.TrimSpace(value) + if strings.HasPrefix(value, "${{") && strings.HasSuffix(value, "}}") { + value = strings.TrimSpace(value[3 : len(value)-2]) + } + if value == "" { + return true, nil + } + if len(value) > 4096 || strings.ContainsRune(value, 0) { + return false, fmt.Errorf("CI condition exceeds 4096 characters or contains a NUL byte") + } + p := conditionParser{rest: value, row: row} + result, err := p.expression(0) + if err != nil || strings.TrimSpace(p.rest) != "" { + return false, fmt.Errorf("cannot statically resolve CI condition %q; unsupported syntax requires review, not a workflow rewrite", value) + } + return conditionTruthy(result), nil +} + +type conditionParser struct { + rest string + row map[string]any + env map[string]string + depth int +} + +func (p *conditionParser) take(token string) bool { + p.rest = strings.TrimSpace(p.rest) + if !strings.HasPrefix(p.rest, token) { + return false + } + p.rest = p.rest[len(token):] + return true +} + +func (p *conditionParser) expression(level int) (any, error) { + if level == 4 { + return p.atom() + } + left, err := p.expression(level + 1) + if err != nil { + return nil, err + } + operators := [][]string{{"||"}, {"&&"}, {"==", "!="}, {"<=", ">=", "<", ">"}}[level] + for { + op := "" + for _, candidate := range operators { + if p.take(candidate) { + op = candidate + break + } + } + if op == "" { + return left, nil + } + right, err := p.expression(level + 1) + if err != nil { + return nil, err + } + switch op { + case "||": + if !conditionTruthy(left) { + left = right + } + case "&&": + if conditionTruthy(left) { + left = right + } + case "==", "!=": + equal, err := conditionEqual(left, right) + if err != nil { + return nil, err + } + left = equal == (op == "==") + default: + left, err = conditionOrder(left, right, op) + if err != nil { + return nil, err + } + } + } +} + +// Support static numbers and strings using GitHub's ordering rules. Two +// strings compare case-insensitively; mixed string/number operands coerce to +// numbers. Invalid numeric strings become NaN, whose comparisons are false. +func conditionOrder(left, right any, op string) (bool, error) { + if symbolicConditionValue(left) || symbolicConditionValue(right) { + return false, fmt.Errorf("action path contents require runtime resolution") + } + var a, b float64 + if x, ok := left.(string); ok { + if y, ok := right.(string); ok { + a = float64(strings.Compare(strings.ToLower(x), strings.ToLower(y))) + return compareConditionNumbers(a, 0, op), nil + } + } + number := func(value any) (float64, error) { + switch v := value.(type) { + case int: + return float64(v), nil + case float64: + return v, nil + case string: + if v == "" { + return 0, nil + } + var n any + if json.Unmarshal([]byte(v), &n) == nil { + if f, ok := n.(float64); ok { + return f, nil + } + } + return math.NaN(), nil + default: + return 0, fmt.Errorf("unsupported relational operand requires review") + } + } + a, err := number(left) + if err != nil { + return false, err + } + b, err = number(right) + if err != nil { + return false, err + } + return compareConditionNumbers(a, b, op), nil +} + +func compareConditionNumbers(a, b float64, op string) bool { + switch op { + case "<": + return a < b + case "<=": + return a <= b + case ">": + return a > b + default: + return a >= b + } +} + +func (p *conditionParser) atom() (any, error) { + p.depth++ + defer func() { p.depth-- }() + if p.depth > 32 { + return nil, fmt.Errorf("condition nesting exceeds 32") + } + if p.take("!") { + value, err := p.atom() + return !conditionTruthy(value), err + } + if p.take("(") { + value, err := p.expression(0) + if err != nil || !p.take(")") { + return nil, fmt.Errorf("unclosed group") + } + return value, nil + } + if p.take("'") { + var value strings.Builder + for len(p.rest) > 0 { + i := strings.IndexByte(p.rest, '\'') + if i < 0 { + break + } + value.WriteString(p.rest[:i]) + p.rest = p.rest[i+1:] + if !strings.HasPrefix(p.rest, "'") { + return value.String(), nil + } + value.WriteByte('\'') + p.rest = p.rest[1:] + } + return nil, fmt.Errorf("unclosed string") + } + p.rest = strings.TrimSpace(p.rest) + i := 0 + for i < len(p.rest) && (unicode.IsLetter(rune(p.rest[i])) || unicode.IsDigit(rune(p.rest[i])) || strings.ContainsRune("._-", rune(p.rest[i]))) { + i++ + } + token := p.rest[:i] + p.rest = p.rest[i:] + switch token { + case "true": + return true, nil + case "false": + return false, nil + case "format": + return p.format() + case "fromJSON": + if !p.take("(") { + return nil, fmt.Errorf("missing JSON argument") + } + value, err := p.expression(0) + if err != nil || !p.take(")") { + return nil, fmt.Errorf("unclosed JSON argument") + } + text, ok := value.(string) + if !ok || symbolicConditionValue(text) { + return nil, fmt.Errorf("JSON argument requires a known string") + } + var values []any + if json.Unmarshal([]byte(text), &values) != nil || values == nil || len(values) > 128 { + return nil, fmt.Errorf("only JSON arrays of at most 128 strings are supported") + } + result := make([]string, len(values)) + for i, value := range values { + text, ok := value.(string) + if !ok || strings.ContainsRune(text, 0) { + return nil, fmt.Errorf("only JSON string array entries are supported") + } + result[i] = text + } + return result, nil + case "contains": + if !p.take("(") { + return nil, fmt.Errorf("missing contains arguments") + } + search, err := p.expression(0) + if err != nil || !p.take(",") { + return nil, fmt.Errorf("missing contains item") + } + value, err := p.expression(0) + if err != nil || !p.take(")") { + return nil, fmt.Errorf("unclosed contains arguments") + } + item, ok := value.(string) + if !ok || symbolicConditionValue(item) || symbolicConditionValue(search) { + return nil, fmt.Errorf("contains requires known string operands") + } + switch values := search.(type) { + case string: + return strings.Contains(strings.ToLower(values), strings.ToLower(item)), nil + case []string: + for _, value := range values { + if strings.EqualFold(value, item) { + return true, nil + } + } + return false, nil + default: + return nil, fmt.Errorf("contains requires a string or string array") + } + case "startsWith": + if !p.take("(") { + return nil, fmt.Errorf("missing argument list") + } + left, err := p.expression(0) + if err != nil || !p.take(",") { + return nil, fmt.Errorf("missing prefix argument") + } + right, err := p.expression(0) + if err != nil || !p.take(")") { + return nil, fmt.Errorf("unclosed function") + } + if symbolicConditionValue(left) || symbolicConditionValue(right) { + return nil, fmt.Errorf("action path contents require runtime resolution") + } + return strings.HasPrefix(strings.ToLower(fmt.Sprint(left)), strings.ToLower(fmt.Sprint(right))), nil + } + if key, ok := strings.CutPrefix(token, "matrix."); ok { + value, exists := p.row[key] + if exists { + if text, ok := value.(string); ok && strings.ContainsRune(text, 0) { + return nil, fmt.Errorf("matrix value contains a NUL byte") + } + return value, nil + } + } + if key, ok := strings.CutPrefix(token, "env."); ok { + if value, exists := p.env[key]; exists { + return value, nil + } + } + if number, err := strconv.ParseFloat(token, 64); err == nil { + return number, nil + } + return nil, fmt.Errorf("unsupported operand %q", token) +} + +func conditionTruthy(value any) bool { + switch v := value.(type) { + case bool: + return v + case string: + return v != "" + case int: + return v != 0 + case float64: + return v != 0 + case []string: + return true + default: + return false + } +} + +func conditionEqual(left, right any) (bool, error) { + // An active action exports nonempty paths; their contents remain unknown. + if (symbolicConditionValue(left) && right == "") || (symbolicConditionValue(right) && left == "") { + return false, nil + } + if symbolicConditionValue(left) || symbolicConditionValue(right) { + return false, fmt.Errorf("action path contents require runtime resolution") + } + if a, ok := left.(string); ok { + if b, ok := right.(string); ok { + return strings.EqualFold(a, b), nil + } + } + if a, ok := left.(bool); ok { + if b, ok := right.(bool); ok { + return a == b, nil + } + } + // Avoid approximating GitHub's loose cross-type coercions. A mixed comparison + // remains unverified rather than selecting the wrong runtime silently. + number := func(v any) (float64, bool) { + switch n := v.(type) { + case int: + return float64(n), true + case float64: + return n, true + } + return 0, false + } + a, aok := number(left) + b, bok := number(right) + if aok && bok { + return a == b, nil + } + return false, fmt.Errorf("mixed-type comparison requires review") +} diff --git a/internal/onboard/runtime_conditions_test.go b/internal/onboard/runtime_conditions_test.go new file mode 100644 index 00000000..d2324630 --- /dev/null +++ b/internal/onboard/runtime_conditions_test.go @@ -0,0 +1,137 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStaticConditionTypesAndPrecedence(t *testing.T) { + row := map[string]any{"node": "20.x", "enabled": false, "text": "false", "number": 22} + for _, tc := range []struct { + expression string + want bool + }{ + {"matrix.node == '18.x' || matrix.node == '20.x' || matrix.node == '22.x'", true}, + {"(matrix.node == '18.x' || matrix.node == '20.x') && !matrix.enabled", true}, + {"false || true && false", false}, + {"(false || true) && false", false}, + {"contains(matrix.node, '20')", true}, + {`contains(fromJSON('["18.x", "20.x", "22.x"]'), matrix.node)`, true}, + {`contains(fromJSON('["18.x", "22.x"]'), matrix.node)`, false}, + {`contains(fromJSON('["20.X"]'), matrix.node)`, true}, + {`contains(fromJSON('[]'), matrix.node)`, false}, + {"matrix.enabled", false}, {"matrix.text", true}, + {"matrix.enabled == false", true}, {"matrix.number == 22", true}, + {"startsWith(matrix.node, '20') && matrix.text != 'FALSE'", false}, + {"'it''s true' == 'IT''S TRUE'", true}, + {"matrix.number >= 22", true}, {"matrix.number > 22", false}, + {"matrix.number <= 22", true}, {"matrix.number < 22", false}, + {"matrix.number >= 18 && matrix.number < 24", true}, + {"matrix.number >= 22 == true", true}, + {"false || matrix.number < 20 && true", false}, + {"'22' >= matrix.number", true}, {"'9' > '22'", true}, + {"'bad' < matrix.number", false}, {"'bad' >= matrix.number", false}, + {"'' < matrix.number", true}, {"'2.2e1' >= matrix.number", true}, + {"'022' >= matrix.number", false}, {"'null' <= matrix.number", false}, + } { + t.Run(tc.expression, func(t *testing.T) { + got, err := runtimeCondition(tc.expression, row) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + for _, expression := range []string{"matrix.enabled == 'false'", "matrix.number == '22'", "false && github.event_name == 'push'", "matrix.node == '20.x' ||", "(true", "true garbage", "matrix.missing"} { + _, err := runtimeCondition(expression, row) + require.Error(t, err, expression) + } + for _, expression := range []string{"matrix.enabled < 22", "matrix.number >=", "matrix.number >= 22 || github.event_name == 'push'", "matrix.number >= 22 garbage", "matrix.missing < 22", `contains(fromJSON('[1]'), matrix.node)`, `fromJSON('{}')`, `fromJSON('null')`, `fromJSON('false')`, `fromJSON('oops')`, `contains(fromJSON(env.NODES), matrix.node)`, `contains(fromJSON('["20.x"]'), github.event_name)`, `contains(matrix.node, 20)`} { + _, err := runtimeCondition(expression, row) + require.Error(t, err, expression) + } +} + +func TestStaticMatrixIncludesFollowOriginalCombinations(t *testing.T) { + // GitHub's documented fruit/animal example: later includes can overwrite + // included values, never original axes, and never augment an appended row. + rows, err := runtimeMatrix(map[string]any{ + "fruit": []any{"apple", "pear"}, "animal": []any{"cat", "dog"}, + "include": []any{ + map[string]any{"color": "green"}, map[string]any{"color": "pink", "animal": "cat"}, + map[string]any{"fruit": "apple", "shape": "circle"}, + map[string]any{"fruit": "banana"}, map[string]any{"fruit": "banana", "animal": "cat"}, + }, + }) + require.NoError(t, err) + require.ElementsMatch(t, []map[string]any{ + {"fruit": "apple", "animal": "cat", "color": "pink", "shape": "circle"}, + {"fruit": "pear", "animal": "cat", "color": "pink"}, + {"fruit": "apple", "animal": "dog", "color": "green", "shape": "circle"}, + {"fruit": "pear", "animal": "dog", "color": "green"}, + {"fruit": "banana"}, {"fruit": "banana", "animal": "cat"}, + }, rows) + rows, err = runtimeMatrix(map[string]any{"node": []any{18, 20, 22}, "os": []any{"linux", "windows"}, "exclude": []any{map[string]any{"node": 18}}, "include": []any{map[string]any{"node": 18, "os": "linux", "instrument": false}}}) + require.NoError(t, err) + require.Len(t, rows, 5) +} + +func TestTSyringeConditionsAndIncludePreserveCoverage(t *testing.T) { + for _, matrix := range []string{ + "node-version: [8.x, 10.x, 12.x, 14.x, 16.x, 18.x, 20.x, 22.x]", + "include: [{node-version: 8.x}, {node-version: 10.x}, {node-version: 12.x}, {node-version: 14.x}, {node-version: 16.x}, {node-version: 18.x}, {node-version: 20.x}, {node-version: 22.x}]", + } { + workflow := fmt.Sprintf(`jobs: + test: + strategy: + matrix: + %s + steps: + - uses: actions/setup-node@v4 + with: {node-version: '${{ matrix.node-version }}'} + - uses: datadog/test-visibility-github-action@v3 + if: contains(fromJSON('["18.x", "20.x", "22.x"]'), matrix.node-version) + with: {languages: js, js-tracer-version: '5.128.0'} + - run: yarn test + env: + NODE_OPTIONS: ${{ contains(fromJSON('["18.x", "20.x", "22.x"]'), matrix.node-version) && format('-r {0}', env.DD_TRACE_PACKAGE) || '' }} + - run: yarn build +`, matrix) + root := newJestRepository(t, workflow) + writeScripts(t, root, map[string]string{"test": "jest", "build": "yarn clean && tsc", "clean": "rimraf ./dist"}) + result := checkCIRuntimes(t.Context(), root, nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "5.128.0", Node: ">=18"}, nil + }) + require.Equal(t, "compatible", result.Status) + require.Len(t, result.Jobs, 8) + for i, job := range result.Jobs { + if i < 5 { + require.Equal(t, "excluded", job.Status) + } else { + require.Equal(t, "compatible", job.Status) + } + } + } +} + +func TestEmptyMatrixDoesNotMasqueradeAsNoCI(t *testing.T) { + workflow := `jobs: + test: + strategy: + matrix: + node: [22] + exclude: [{node: 22}] + steps: + - run: yarn test + env: {NODE_OPTIONS: "-r ${{ env.DD_TRACE_PACKAGE }}"} +` + result := checkCIRuntimes(t.Context(), newJestRepository(t, workflow), nil, nil) + require.Equal(t, "inconclusive", result.Status) + require.Contains(t, result.Jobs[0].Reason, "no entries") +} diff --git a/internal/onboard/runtime_node.go b/internal/onboard/runtime_node.go new file mode 100644 index 00000000..921c0d0f --- /dev/null +++ b/internal/onboard/runtime_node.go @@ -0,0 +1,181 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "slices" + "strconv" + "strings" + "time" +) + +const nodeVersionsManifest = "https://raw.githubusercontent.com/actions/node-versions/main/versions-manifest.json" +const nodeDistributionIndex = "https://nodejs.org/dist/index.json" + +// NodeResolution records a time-dependent alias resolution. setup-node resolves +// LTS aliases to a major, then may use a cached patch: an exact patch is unknown. +type NodeResolution struct { + Version string `json:"version"` + Source string `json:"source"` + ResolvedAt time.Time `json:"resolved_at"` + Platform string `json:"platform,omitempty"` +} + +type nodeResolver func(context.Context, string, string) (NodeResolution, error) + +// Fetch once per check, including on failure, so every job uses the same snapshot. +func setupNodeResolver(client *http.Client) nodeResolver { + type snapshot struct { + data []byte + err error + at time.Time + } + cache := map[string]snapshot{} + return func(ctx context.Context, alias, platform string) (NodeResolution, error) { + source := nodeVersionsManifest + if latestNodeAlias(alias) { + if platform == "" { + return NodeResolution{}, fmt.Errorf("cannot determine the hosted runner platform/architecture for %q", alias) + } + source = nodeDistributionIndex + } + value, ok := cache[source] + if !ok { + value.data, value.err = fetchRuntimeMetadata(ctx, client, source) + value.at = time.Now().UTC() + cache[source] = value + } + if value.err != nil { + return NodeResolution{}, value.err + } + result := NodeResolution{Source: source, ResolvedAt: value.at} + var err error + if latestNodeAlias(alias) { + result.Platform = platform + result.Version, err = resolveLatestNode(value.data, platform) + } else { + result.Version, err = resolveLTSMajor(value.data, alias) + } + return result, err + } +} + +func latestNodeAlias(alias string) bool { + return slices.Contains([]string{"current", "latest", "node"}, alias) +} + +// setup-node selects the newest distribution available for the runner's OS/arch. +func resolveLatestNode(data []byte, platform string) (string, error) { + var releases []struct { + Version string `json:"version"` + Files []string `json:"files"` + } + if err := json.Unmarshal(data, &releases); err != nil { + return "", fmt.Errorf("decode Node distribution index: %w", err) + } + var best [3]int + var selected string + for _, release := range releases { + version, precision, ok := nodeInterval(release.Version) + if ok && precision == 3 && slices.Contains(release.Files, platform) && slices.Compare(version[:], best[:]) > 0 { + best, selected = version, strings.TrimPrefix(release.Version, "v") + } + } + if selected == "" { + return "", fmt.Errorf("node distribution index has no release for %q", platform) + } + return selected, nil +} + +// Unknown/self-hosted platforms stay unverified. macOS architecture must be +// explicit because the default differs between hosted labels and repository types. +func nodeDistributionPlatform(runner any, architecture string, row map[string]any) string { + label, ok := runner.(string) + if !ok { + return "" + } + label, err := runtimeValue(label, row) + if err != nil { + return "" + } + arch, err := runtimeValue(architecture, row) + if err != nil { + return "" + } + var prefix, suffix string + switch { + case strings.HasPrefix(label, "ubuntu-"): + prefix = "linux-" + case strings.HasPrefix(label, "windows-"): + prefix, suffix = "win-", "-exe" + case strings.HasPrefix(label, "macos-"): + if arch == "" { + return "" + } + prefix, suffix = "osx-", "-tar" + default: + return "" + } + if arch == "" { + arch = "x64" + if strings.HasSuffix(label, "-arm") { + arch = "arm64" + } + } + if !slices.Contains([]string{"x64", "x86", "arm64", "arm"}, arch) { + return "" + } + if arch == "arm" { + arch = "armv7l" + } + return prefix + arch + suffix +} + +func resolveLTSMajor(data []byte, alias string) (string, error) { + var releases []struct { + Version string `json:"version"` + Stable bool `json:"stable"` + LTS string `json:"lts"` + } + if err := json.Unmarshal(data, &releases); err != nil { + return "", fmt.Errorf("decode setup-node version manifest: %w", err) + } + // Match setup-node's lts/*, lts/, and lts/-n selectors. + selector := strings.ToLower(strings.TrimPrefix(alias, "lts/")) + majors := map[string]int{} + for _, release := range releases { + version, precision, ok := nodeInterval(release.Version) + if !release.Stable || release.LTS == "" || !ok || precision != 3 { + continue + } + name := strings.ToLower(release.LTS) + majors[name] = max(majors[name], version[0]) + } + if major, ok := majors[selector]; ok { + return strconv.Itoa(major), nil + } + index := 0 + if selector != "*" { + var err error + index, err = strconv.Atoi(strings.TrimPrefix(selector, "-")) + if !strings.HasPrefix(selector, "-") || err != nil || index < 0 { + return "", fmt.Errorf("unknown setup-node LTS alias %q", alias) + } + } + versions := make([]int, 0, len(majors)) + for _, version := range majors { + versions = append(versions, version) + } + slices.Sort(versions) + if index >= len(versions) { + return "", fmt.Errorf("setup-node manifest has no release for %q", alias) + } + return strconv.Itoa(versions[len(versions)-1-index]), nil +} diff --git a/internal/onboard/runtime_node_test.go b/internal/onboard/runtime_node_test.go new file mode 100644 index 00000000..6800f898 --- /dev/null +++ b/internal/onboard/runtime_node_test.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ltsManifestFixture = `[ + {"version":"22.4.1","stable":true,"lts":"Jod"}, + {"version":"26.0.0","stable":true}, + {"version":"24.3.0","stable":true,"lts":"Krypton"}, + {"version":"22.8.0","stable":true,"lts":"Jod"}, + {"version":"28.0.0","stable":false,"lts":"Future"} +]` + +func TestLTSMajorResolution(t *testing.T) { + for _, tc := range []struct{ alias, version string }{ + {"lts/*", "24"}, {"lts/-0", "24"}, {"lts/-1", "22"}, {"lts/Jod", "22"}, + {"lts/krypton", "24"}, {"lts/-2", ""}, {"lts/unknown", ""}, {"lts/-", ""}, {"lts/--1", ""}, + } { + t.Run(tc.alias, func(t *testing.T) { + version, err := resolveLTSMajor([]byte(ltsManifestFixture), tc.alias) + if tc.version == "" { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.version, version) + } + }) + } + for _, data := range []string{"invalid json", "[]", `[{"stable":true,"lts":"Bad","version":"24.x"}]`} { + _, err := resolveLTSMajor([]byte(data), "lts/*") + require.Error(t, err) + } +} + +func TestCILTSResolutionPreservesAliasProvenanceAndUnknownPatch(t *testing.T) { + for _, tc := range []struct{ requirement, status string }{ + {">=22", "compatible"}, {">=26", "incompatible"}, {">=24.2.0", "inconclusive"}, + } { + t.Run(tc.requirement, func(t *testing.T) { + calls := 0 + client := &http.Client{Transport: runtimeTransport(func(request *http.Request) (*http.Response, error) { + calls++ + require.Equal(t, nodeVersionsManifest, request.URL.String()) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(ltsManifestFixture))}, nil + })} + workflow := strings.Replace(fmt.Sprintf(runtimeWorkflow, ""), "node-version: ${{ matrix.node-version }}", "node-version: lts/*", 1) + result := checkCIRuntimes(t.Context(), newJestRepository(t, workflow), setupNodeResolver(client), func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.16.0", Node: tc.requirement}, nil + }) + require.Equal(t, tc.status, result.Status) + require.Equal(t, 1, calls) + require.Len(t, result.Jobs, 4) + for _, job := range result.Jobs { + require.Equal(t, "lts/*", job.Node) + require.Equal(t, "24", job.NodeResolution.Version) + require.Equal(t, nodeVersionsManifest, job.NodeResolution.Source) + require.False(t, job.NodeResolution.ResolvedAt.IsZero()) + } + }) + } +} + +func TestUnavailableLTSMetadataStaysInconclusive(t *testing.T) { + calls := 0 + client := &http.Client{Transport: runtimeTransport(func(*http.Request) (*http.Response, error) { + calls++ + return nil, fmt.Errorf("metadata unavailable") + })} + workflow := strings.Replace(fmt.Sprintf(runtimeWorkflow, ""), "node-version: ${{ matrix.node-version }}", "node-version: lts/*", 1) + result := checkCIRuntimes(t.Context(), newJestRepository(t, workflow), setupNodeResolver(client), func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.16.0", Node: ">=22"}, nil + }) + require.Equal(t, "inconclusive", result.Status) + require.Equal(t, 1, calls) + for _, job := range result.Jobs { + require.Nil(t, job.NodeResolution) + require.Contains(t, job.Reason, "metadata unavailable") + } +} diff --git a/internal/onboard/runtime_test.go b/internal/onboard/runtime_test.go new file mode 100644 index 00000000..9d0cbc0e --- /dev/null +++ b/internal/onboard/runtime_test.go @@ -0,0 +1,233 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const runtimeWorkflow = `jobs: + tests: + strategy: + matrix: + node-version: [20.20.1, 22.22.1, 24.14.0, 25.8.1] + steps: + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - uses: datadog/test-visibility-github-action@v3 + %s + with: + languages: js + - run: npm run test + env: {NODE_OPTIONS: "-r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}"} +` + +func TestCIRuntimesCatchLuxonRegressionAndRespectExclusion(t *testing.T) { + for _, tc := range []struct{ name, guard, status string }{ + {"unconditional instrumentation", "", "incompatible"}, + {"preserve Node 20 tests", "if: ${{ !startsWith(matrix.node-version, '20.') }}", "compatible"}, + } { + t.Run(tc.name, func(t *testing.T) { + workflow := fmt.Sprintf(runtimeWorkflow, tc.guard) + root := newJestRepository(t, workflow) + calls := 0 + result := checkCIRuntimes(t.Context(), root, nil, func(_ context.Context, action, version string) (tracerRequirement, error) { + calls++ + require.Equal(t, githubAction+"@v3", action) + require.Empty(t, version) + return tracerRequirement{Version: "6.16.0", Node: ">=22"}, nil + }) + require.Equal(t, tc.status, result.Status) + require.Len(t, result.Jobs, 4) + require.Equal(t, 1, calls) + require.Equal(t, "20.20.1", result.Jobs[0].Node) + if tc.status == "incompatible" { + require.Equal(t, "incompatible", result.Jobs[0].Status) + require.Equal(t, "dd-trace@6.16.0", result.Jobs[0].Tracer) + require.Equal(t, ">=22", result.Jobs[0].Requirement) + } else { + require.Equal(t, "excluded", result.Jobs[0].Status) + } + data, err := os.ReadFile(filepath.Join(root, ".github/workflows/test.yml")) + require.NoError(t, err) + require.Equal(t, workflow, string(data)) + }) + } +} + +func TestCIRuntimesNeverGuessUnknownConfigurations(t *testing.T) { + original := fmt.Sprintf(runtimeWorkflow, "") + for _, tc := range []struct{ name, old, new, reason string }{ + {"dynamic matrix", "node-version: [20.20.1, 22.22.1, 24.14.0, 25.8.1]", "${{ fromJSON(needs.build.outputs.matrix) }}", "dynamic CI matrix"}, + {"dynamic version", "node-version: ${{ matrix.node-version }}", "node-version: ${{ inputs.node }}", "cannot statically resolve"}, + {"version file", "node-version: ${{ matrix.node-version }}", "node-version-file: .nvmrc", "missing or dynamic"}, + {"lts alias", "node-version: ${{ matrix.node-version }}", "node-version: lts/*", "missing or dynamic"}, + {"unknown condition", "with:\n languages: js", "if: ${{ github.event_name == 'push' }}\n with:\n languages: js", "cannot statically resolve CI condition"}, + {"no action", "uses: datadog/test-visibility-github-action@v3", "uses: actions/checkout@v3", "No Datadog JavaScript action"}, + {"other language", "languages: js", "languages: python", "No Datadog JavaScript action"}, + } { + t.Run(tc.name, func(t *testing.T) { + result := checkCIRuntimes(t.Context(), newJestRepository(t, strings.Replace(original, tc.old, tc.new, 1)), nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.16.0", Node: ">=22"}, nil + }) + want := "inconclusive" + if tc.reason == "No Datadog JavaScript action" { + want = "incompatible" + require.Equal(t, "missing_instrumentation", result.Jobs[0].Code) + } + require.Equal(t, want, result.Status) + require.Contains(t, result.Jobs[0].Reason, tc.reason) + }) + } + result := checkCIRuntimes(t.Context(), newJestRepository(t, original), nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{}, fmt.Errorf("metadata unavailable") + }) + require.Equal(t, "inconclusive", result.Status) + require.Contains(t, result.Jobs[0].Reason, "metadata unavailable") + result = checkCIRuntimes(t.Context(), t.TempDir(), nil, nil) + require.Equal(t, "not applicable", result.Status) +} + +func TestCIRuntimesUseSelectedTracerAndEveryJob(t *testing.T) { + workflow := `jobs: + old: + steps: + - uses: actions/setup-node@v4 + with: {node-version: '20'} + - uses: datadog/test-visibility-github-action@v3 + with: {languages: js, js-tracer-version: 5.99.0} + - run: npm test + env: {NODE_OPTIONS: "-r ${{ env.DD_TRACE_PACKAGE }}"} + new: + steps: + - uses: actions/setup-node@v4 + with: {node-version: '22'} + - uses: datadog/test-visibility-github-action@v4 + with: {languages: js} + - run: npm test + env: {NODE_OPTIONS: "-r ${{ env.DD_TRACE_PACKAGE }}"} +` + result := checkCIRuntimes(t.Context(), newJestRepository(t, workflow), nil, func(_ context.Context, action, version string) (tracerRequirement, error) { + if version == "5.99.0" { + return tracerRequirement{Version: version, Node: ">=18"}, nil + } + require.Equal(t, githubAction+"@v4", action) + return tracerRequirement{Version: "7.0.0", Node: ">=24"}, nil + }) + require.Equal(t, "incompatible", result.Status) + require.Len(t, result.Jobs, 2) + require.Equal(t, "incompatible", result.Jobs[0].Status) + require.Equal(t, "compatible", result.Jobs[1].Status) +} + +func TestNodeRequirements(t *testing.T) { + for _, tc := range []struct{ node, engine, status string }{ + {"20.20.1", ">=22", "incompatible"}, {"20", ">=22", "incompatible"}, {"20.x", ">=22", "incompatible"}, + {"22", ">=22", "compatible"}, {"v22.0.0", ">=22", "compatible"}, {"24.1.2", ">=22", "compatible"}, + {"22", ">=24", "incompatible"}, {"22", ">=22.2.0", "inconclusive"}, {"22.1", ">=22.2.0", "incompatible"}, + {"22.2", ">=22.2.0", "compatible"}, {"lts/*", ">=22", "inconclusive"}, {"", ">=22", "inconclusive"}, + {"22", ">=18 <23", "inconclusive"}, {"22", "", "inconclusive"}, {"22", ">=22 || >=24", "inconclusive"}, + {"22.x.3", ">=22", "inconclusive"}, + } { + t.Run(tc.node+"/"+tc.engine, func(t *testing.T) { + status, _ := CompareNodeRequirement(tc.node, tc.engine) + require.Equal(t, tc.status, status) + }) + } +} + +func TestRuntimeMatrixAndConditions(t *testing.T) { + rows, err := runtimeMatrix(map[string]any{"node": []any{"20", "22"}, "os": []any{"linux", "windows"}}) + require.NoError(t, err) + require.Len(t, rows, 4) + for _, tc := range []struct { + value string + want bool + }{ + {"", true}, {"false", false}, {"matrix.node == '22'", true}, {"${{ matrix.node != '20' }}", true}, + {"${{ startsWith(matrix.node, '2') }}", true}, {"${{ !startsWith(matrix.node, '2') }}", false}, + } { + actual, err := runtimeCondition(tc.value, map[string]any{"node": "22"}) + require.NoError(t, err) + require.Equal(t, tc.want, actual) + } + _, err = runtimeCondition("matrix.missing != '20'", map[string]any{"node": "22"}) + require.Error(t, err) + _, err = runtimeMatrix(map[string]any{"node": []any{"${{ inputs.node }}"}}) + require.Error(t, err) + _, err = runtimeMatrix(map[string]any{"node": make([]any, 257)}) + require.Error(t, err) +} + +type runtimeTransport func(*http.Request) (*http.Response, error) + +func (f runtimeTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestRuntimeMetadataUsesActionRefDefaultAndExplicitOverride(t *testing.T) { + for _, explicit := range []string{"", "5.99.0"} { + t.Run(explicit, func(t *testing.T) { + var addresses []string + client := &http.Client{Transport: runtimeTransport(func(r *http.Request) (*http.Response, error) { + addresses = append(addresses, r.URL.String()) + body := `{"version":"6.16.0","engines":{"node":">=22"}}` + if r.URL.Host == "raw.githubusercontent.com" { + body = "inputs:\n js-tracer-version:\n default: '6.16.0'\n" + } + if strings.HasSuffix(r.URL.Path, "/5.99.0") { + body = `{"version":"5.99.0","engines":{"node":">=18"}}` + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil + })} + requirement, err := resolveRequirement(t.Context(), client, githubAction+"@abc123", explicit) + require.NoError(t, err) + if explicit == "" { + require.Equal(t, []string{"https://raw.githubusercontent.com/DataDog/test-visibility-github-action/abc123/action.yml", "https://registry.npmjs.org/dd-trace/6.16.0"}, addresses) + require.Equal(t, ">=22", requirement.Node) + } else { + require.Equal(t, []string{"https://registry.npmjs.org/dd-trace/5.99.0"}, addresses) + require.Equal(t, ">=18", requirement.Node) + } + }) + } +} + +func TestRuntimeMetadataErrorsAreNotCompatibility(t *testing.T) { + for _, tc := range []struct { + name, body string + status int + }{ + {"not found", "", 404}, {"malformed", "not json", 200}, {"missing engines", `{"version":"6.16.0"}`, 200}, {"oversized", strings.Repeat("x", (1<<20)+1), 200}, + } { + t.Run(tc.name, func(t *testing.T) { + client := &http.Client{Transport: runtimeTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: tc.status, Body: io.NopCloser(strings.NewReader(tc.body))}, nil + })} + _, err := resolveRequirement(t.Context(), client, githubAction+"@v3", "6.16.0") + require.Error(t, err) + }) + } +} + +func TestExplicitEmptyTracerInputOverridesActionDefault(t *testing.T) { + workflow := strings.Replace(fmt.Sprintf(runtimeWorkflow, ""), "languages: js", "languages: js\n js-tracer-version: ''", 1) + result := checkCIRuntimes(t.Context(), newJestRepository(t, workflow), nil, func(_ context.Context, _ string, version string) (tracerRequirement, error) { + require.Equal(t, "latest", version) + return tracerRequirement{Version: "6.16.0", Node: ">=22", Requested: version}, nil + }) + require.Equal(t, "incompatible", result.Status) + require.Equal(t, "latest", result.Jobs[0].TracerRequested) + require.True(t, result.Jobs[0].TracerFloating) +} diff --git a/internal/onboard/runtime_values.go b/internal/onboard/runtime_values.go new file mode 100644 index 00000000..d5d9b485 --- /dev/null +++ b/internal/onboard/runtime_values.go @@ -0,0 +1,209 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "fmt" + "maps" + "reflect" + "regexp" + "slices" + "strconv" + "strings" +) + +// Expand only bounded, scalar matrices. Includes augment original combinations; +// rows introduced by an include are never candidates for subsequent includes. +// https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations +func runtimeMatrix(value any) ([]map[string]any, error) { + rows := []map[string]any{{}} + if value == nil { + return rows, nil + } + matrix, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("dynamic CI matrix cannot be resolved statically") + } + axes := 0 + for _, key := range slices.Sorted(maps.Keys(matrix)) { + if key == "include" || key == "exclude" { + continue + } + axes++ + values, ok := matrix[key].([]any) + if !ok || len(values) == 0 || len(rows)*len(values) > 256 { + return nil, fmt.Errorf("matrix axis %s is not a bounded static list", key) + } + var expanded []map[string]any + for _, row := range rows { + for _, value := range values { + if !staticScalar(value) { + return nil, fmt.Errorf("matrix axis %s contains an expression or non-scalar value", key) + } + next := maps.Clone(row) + next[key] = value + expanded = append(expanded, next) + } + } + rows = expanded + } + exclusions, err := matrixEntries(matrix, "exclude") + if err != nil { + return nil, err + } + rows = slices.DeleteFunc(rows, func(row map[string]any) bool { + for _, exclude := range exclusions { + match := true + for key, value := range exclude { + actual, ok := row[key] + match = match && ok && reflect.DeepEqual(actual, value) + } + if match { + return true + } + } + return false + }) + if axes == 0 { + rows = nil + } + originals := make([]map[string]any, len(rows)) + for i, row := range rows { + originals[i] = maps.Clone(row) + } + includes, err := matrixEntries(matrix, "include") + if err != nil { + return nil, err + } + for _, include := range includes { + matched := false + for i, original := range originals { + compatible := true + for key, value := range include { + if actual, exists := original[key]; exists && !reflect.DeepEqual(actual, value) { + compatible = false + } + } + if compatible { + maps.Copy(rows[i], include) + matched = true + } + } + if !matched { + rows = append(rows, maps.Clone(include)) + } + if len(rows) > 256 { + return nil, fmt.Errorf("matrix expands beyond 256 entries") + } + } + return rows, nil +} + +func staticScalar(value any) bool { + switch v := value.(type) { + case string: + return !strings.Contains(v, "${{") + case int, float64, bool: + return true + default: + return false + } +} + +func matrixEntries(matrix map[string]any, name string) ([]map[string]any, error) { + value, exists := matrix[name] + if !exists { + return nil, nil + } + entries, ok := value.([]any) + if !ok || len(entries) > 256 { + return nil, fmt.Errorf("matrix %s must be a bounded static list", name) + } + result := make([]map[string]any, 0, len(entries)) + for _, entry := range entries { + row, ok := entry.(map[string]any) + if !ok || len(row) == 0 { + return nil, fmt.Errorf("matrix %s requires nonempty scalar objects", name) + } + for _, v := range row { + if !staticScalar(v) { + return nil, fmt.Errorf("matrix %s contains a dynamic or non-scalar value", name) + } + } + result = append(result, row) + } + return result, nil +} + +var matrixReference = regexp.MustCompile(`^\$\{\{\s*matrix\.([a-zA-Z0-9_-]+)\s*\}\}$`) + +func runtimeValue(value string, row map[string]any) (string, error) { + if match := matrixReference.FindStringSubmatch(strings.TrimSpace(value)); match != nil { + resolved, ok := row[match[1]] + if !ok { + return "", fmt.Errorf("unknown matrix reference %s", value) + } + return fmt.Sprint(resolved), nil + } + if strings.Contains(value, "${{") { + return "", fmt.Errorf("cannot statically resolve %s", value) + } + return strings.TrimSpace(value), nil +} + +var numericNode = regexp.MustCompile(`^v?([0-9]+)(?:\.([0-9]+|x|\*))?(?:\.([0-9]+|x|\*))?$`) +var minimumEngine = regexp.MustCompile(`^>=\s*([0-9]+(?:\.[0-9]+){0,2})$`) + +func nodeInterval(value string) ([3]int, int, bool) { + var version [3]int + match := numericNode.FindStringSubmatch(value) + if match == nil { + return version, 0, false + } + precision := 0 + for i, part := range match[1:] { + if part == "" || part == "x" || part == "*" { + continue + } + if precision != i { + return version, 0, false + } + number, err := strconv.Atoi(part) + if err != nil || number > 1000000 { + return version, 0, false + } + version[i] = number + precision++ + } + return version, precision, true +} + +// CompareNodeRequirement checks a static Node version against a tracer minimum. +func CompareNodeRequirement(node, requirement string) (string, string) { + minimum := minimumEngine.FindStringSubmatch(requirement) + if minimum == nil { + return "inconclusive", fmt.Sprintf("Unsupported Node engine range %q; no compatibility assumption was made.", requirement) + } + required, _, ok := nodeInterval(minimum[1]) + if !ok { + return "inconclusive", "Cannot parse tracer Node requirement." + } + lower, precision, ok := nodeInterval(node) + if !ok { + return "inconclusive", fmt.Sprintf("Node version %q is missing or dynamic; use an explicit setup-node version to check it.", node) + } + if slices.Compare(lower[:], required[:]) >= 0 { + return "compatible", fmt.Sprintf("Node %s satisfies %s.", node, requirement) + } + if precision < 3 { + upper := lower + upper[precision-1]++ + if slices.Compare(upper[:], required[:]) > 0 { + return "inconclusive", fmt.Sprintf("Node %s can resolve below or above %s; use an exact version.", node, requirement) + } + } + return "incompatible", fmt.Sprintf("Node %s does not satisfy %s. Keep this test coverage, but exclude this runtime from instrumentation or choose a compatible tracer/runtime.", node, requirement) +} diff --git a/internal/onboard/scope.go b/internal/onboard/scope.go new file mode 100644 index 00000000..00b9685b --- /dev/null +++ b/internal/onboard/scope.go @@ -0,0 +1,195 @@ +package onboard + +import ( + "fmt" + "io" + "maps" + "path/filepath" + "slices" +) + +// TestCommand identifies an explicit framework invocation, not the surrounding +// build, publication or shell script. Locations retain every CI consumer. +type TestCommand struct { + Directory string `json:"directory"` + Command string `json:"command"` + Environment map[string]string `json:"environment,omitempty"` + Locations []string `json:"locations,omitempty"` + Prerequisites []BuildCommand `json:"prerequisites,omitempty"` + UnvalidatedSetup []string `json:"unvalidated_setup,omitempty"` +} + +// BuildCommand is a statically resolved build preceding a test in the same +// command chain. It is executed without test instrumentation. +type BuildCommand struct { + Environment map[string]string `json:"environment,omitempty"` + Kind string `json:"kind,omitempty"` + Directory string `json:"directory"` + Command string `json:"command"` +} + +type ValidationScope struct { + Commands []TestCommand `json:"commands"` + Unresolved []string `json:"unresolved,omitempty"` + Review []string `json:"review,omitempty"` +} + +func yarnImplicitInstall(words []string) bool { + if len(words) == 0 || words[0] != "yarn" { + return false + } + for _, word := range words[1:] { + if !slices.Contains([]string{"--frozen-lockfile", "--non-interactive", "--ignore-scripts", "--ignore-optional", "--offline", "--immutable"}, word) { + return false + } + } + return true +} + +// DiscoverValidationScope reads CI and package aliases without running them. +// Unknown fragments stay visible alongside known test invocations. +func DiscoverValidationScope(root, framework string) (ValidationScope, error) { + var scope ValidationScope + workflows, err := readWorkflows(root) + if err != nil { + return scope, err + } + for _, workflow := range workflows { + for _, name := range slices.Sorted(maps.Keys(workflow.Jobs)) { + job := workflow.Jobs[name] + rows, err := runtimeMatrix(job.Strategy.Matrix) + if err != nil { + scope.Unresolved = append(scope.Unresolved, workflow.Path+" / "+name+": "+err.Error()) + continue + } + for _, row := range rows { + active, err := runtimeCondition(job.If, row) + if err != nil { + scope.Unresolved = append(scope.Unresolved, workflow.Path+" / "+name+": "+err.Error()) + continue + } + if !active { + continue + } + var pendingSetup []string + var preparation []BuildCommand + var unresolved, review []string + var commands []TestCommand + for i, step := range job.Steps { + active, conditionErr := stepCondition(step, row) + if conditionErr == nil && !active { + continue + } + jobLabel := name + if len(row) > 0 { + jobLabel += fmt.Sprint(" ", row) + } + location := fmt.Sprintf("%s / %s / step %d (%s)", workflow.Path, jobLabel, stepNumber(step, i), step.Run) + resolved, env, envErr := resolveVariantStep(workflow, job, step, row) + resolution := resolveTestStep(root, workflow, job, resolved, "javascript", framework) + if conditionErr != nil { + pendingSetup = append(pendingSetup, conditionErr.Error()) + resolution.Reason = conditionErr.Error() + } + if envErr != nil { + pendingSetup = append(pendingSetup, envErr.Error()) + resolution.Reason = envErr.Error() + } + if resolution.Review { + for j := range resolution.Builds { + resolution.Builds[j].Environment = maps.Clone(env) + } + preparation = append(preparation, resolution.Builds...) + review = append(review, location+": "+resolution.Reason) + continue + } + if resolution.Reason != "" { + unresolved = append(unresolved, location+": "+resolution.Reason) + } + if resolution.ReviewReason != "" { + review = append(review, location+": "+resolution.ReviewReason) + } + for j := range resolution.Builds { + resolution.Builds[j].Environment = maps.Clone(env) + } + for _, entry := range resolution.Tests { + for j := range entry.Prerequisites { + entry.Prerequisites[j].Environment = maps.Clone(env) + } + entry.Prerequisites = append(slices.Clone(preparation), entry.Prerequisites...) + entry.UnvalidatedSetup = append(slices.Clone(pendingSetup), entry.UnvalidatedSetup...) + entry.Environment = env + entry.Directory = filepath.ToSlash(filepath.Clean(entry.Directory)) + entry.Locations = []string{location} + commands = append(commands, entry) + } + if resolution.Reason != "" { + pendingSetup = append(pendingSetup, location+": "+resolution.Reason) + } + if !resolution.Matched { + preparation = append(preparation, resolution.Builds...) + } + } + scope.Unresolved = append(scope.Unresolved, unresolved...) + scope.Review = append(scope.Review, review...) + for _, entry := range commands { + index := slices.IndexFunc(scope.Commands, func(previous TestCommand) bool { + return previous.Directory == entry.Directory && previous.Command == entry.Command && slices.EqualFunc(previous.Prerequisites, entry.Prerequisites, EqualPreparation) && slices.Equal(previous.UnvalidatedSetup, entry.UnvalidatedSetup) && maps.Equal(previous.Environment, entry.Environment) + }) + if index < 0 { + scope.Commands = append(scope.Commands, entry) + } else { + scope.Commands[index].Locations = append(scope.Commands[index].Locations, entry.Locations...) + } + } + } + } + } + return scope, nil +} + +// ValidationCommands also resolves a selected local alias, for scope accounting. +func ValidationCommands(root, command, framework string) []TestCommand { + remaining := 256 + return resolveJestSequence(root, root, command, nil, &remaining, framework).Tests +} + +func WriteValidationScope(output io.Writer, scope ValidationScope) { + _, _ = fmt.Fprintln(output, "\nRequired local validation configurations (every CI consumer must also be instrumented):") + for _, entry := range scope.Commands { + _, _ = fmt.Fprintf(output, " - [%s] %s\n", entry.Directory, entry.Command) + for _, build := range entry.Prerequisites { + _, _ = fmt.Fprintf(output, " Preparation: [%s] %s\n", build.Directory, build.Command) + } + for _, setup := range entry.UnvalidatedSetup { + _, _ = fmt.Fprintf(output, " Unvalidated setup: %s\n", setup) + } + for _, location := range entry.Locations { + _, _ = fmt.Fprintf(output, " %s\n", location) + } + } + _, _ = fmt.Fprintln(output, "--all replays listed preparation in separate temporary copies before each comparison, without instrumentation. --check-only executes no preparation. Unresolved setup stays unvalidated. Preserve CI commands.") + for _, reason := range scope.Unresolved { + _, _ = fmt.Fprintf(output, " - Unresolved scope: %s\n", reason) + } + for _, reason := range scope.Review { + _, _ = fmt.Fprintf(output, " - Review separately: %s\n", reason) + } +} + +// File rewrites in earlier CI steps change what the same test command executes. +func mutatesCIFiles(command string) bool { + commands, err := staticCommands(command) + if err != nil { + return false + } // Already retained as unresolved setup. + for _, words := range commands { + if len(words) > 0 && slices.Contains([]string{"sed", "cp", "mv", "rm", "patch"}, words[0]) { + return true + } + if len(words) > 1 && slices.Contains([]string{"pnpm", "yarn", "bun"}, words[0]) && words[1] == "add" { + return true + } + } + return false +} diff --git a/internal/onboard/scope_test.go b/internal/onboard/scope_test.go new file mode 100644 index 00000000..38bc1d86 --- /dev/null +++ b/internal/onboard/scope_test.go @@ -0,0 +1,238 @@ +package onboard + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func immerScopeFixture(t *testing.T) string { + t.Helper() + root := newJestRepository(t, `jobs: + test: + steps: + - uses: actions/setup-node@v4 + with: {node-version: '24'} + - run: yarn --frozen-lockfile + - uses: datadog/test-visibility-github-action@v3 + with: {languages: js, js-tracer-version: '6.18.0'} + - run: yarn test + env: {NODE_OPTIONS: '-r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}'} + - run: yarn coverage + env: {NODE_OPTIONS: '-r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}'} + - run: yarn test:perf + release: + steps: + - uses: actions/setup-node@v4 + with: {node-version: '24'} + - run: yarn test + docs: + steps: + - run: cd website && yarn && yarn build +`) + writeScripts(t, root, map[string]string{"test": "vitest run && yarn test:build && yarn test:flow", "test:build": "yarn build && vitest run --config vitest.config.build.ts", "test:flow": "yarn flow check tests/flow", "build": "tsup", "coverage": "vitest run --coverage", "test:perf": "cd performance && node bench.mjs"}) + writeScripts(t, filepath.Join(root, "website"), map[string]string{"build": "docusaurus build"}) + require.NoError(t, os.Mkdir(filepath.Join(root, "performance"), 0755)) + return root +} + +func TestScopeRetainsEveryImmerConfigurationAndConsumer(t *testing.T) { + root := immerScopeFixture(t) + scope, err := DiscoverValidationScope(root, "vitest") + require.NoError(t, err) + require.Empty(t, scope.Unresolved) + require.Len(t, scope.Commands, 3) + require.Equal(t, "vitest run", scope.Commands[0].Command) + require.Equal(t, "vitest run --config vitest.config.build.ts", scope.Commands[1].Command) + require.Equal(t, "vitest run --coverage", scope.Commands[2].Command) + require.Empty(t, scope.Commands[0].Prerequisites) + require.Equal(t, []BuildCommand{{Directory: ".", Command: "tsup"}}, scope.Commands[1].Prerequisites) + require.Empty(t, scope.Commands[2].Prerequisites) + require.Len(t, scope.Commands[0].Locations, 2) + require.Len(t, scope.Commands[1].Locations, 2) + require.Contains(t, strings.Join(scope.Review, "\n"), "node bench.mjs") + result := checkCIRuntimes(t.Context(), root, nil, func(context.Context, string, string) (tracerRequirement, error) { + return tracerRequirement{Version: "6.18.0", Node: ">=22"}, nil + }, "vitest") + require.Equal(t, "incompatible", result.Status) + var missing []RuntimeFinding + for _, finding := range result.Jobs { + if finding.Code == "missing_instrumentation" { + missing = append(missing, finding) + } + } + require.Len(t, missing, 1) + require.Equal(t, "release", missing[0].Job) +} + +func TestBuildPrerequisitesKeepOrderAndDistinctConfigurations(t *testing.T) { + for _, framework := range []string{"jest", "vitest"} { + t.Run(framework, func(t *testing.T) { + root := newJestRepository(t, `jobs: + test: + steps: + - run: npm run test:a + - run: npm run test:b + - run: npm run test:a +`) + writeScripts(t, root, map[string]string{ + "test:a": "yarn build:a && " + framework + " run && yarn build:after", + "test:b": "yarn build:b && " + framework + " run", + "build:a": "tsc && tsup --config a.ts", "build:b": "tsup --config b.ts", "build:after": "rollup", + }) + scope, err := DiscoverValidationScope(root, framework) + require.NoError(t, err) + require.Len(t, scope.Commands, 2, "same test command with different builds is a different configuration") + require.Len(t, scope.Commands[0].Locations, 2) + require.Equal(t, []BuildCommand{{Directory: ".", Command: "tsc"}, {Directory: ".", Command: "tsup --config a.ts"}}, scope.Commands[0].Prerequisites) + require.Equal(t, []BuildCommand{{Directory: ".", Command: "tsup --config b.ts"}}, scope.Commands[1].Prerequisites) + selected := ValidationCommands(root, "MODE=production npm run test:a", framework) + require.Equal(t, "MODE=production tsc", selected[0].Prerequisites[0].Command, "must not silently discard the alias environment") + }) + } +} + +func TestPartialResolutionDoesNotHideMissingInstrumentation(t *testing.T) { + root := newJestRepository(t, `jobs: + test: + steps: + - run: npm test +`) + writeScripts(t, root, map[string]string{"test": "node unknown-setup.js && vitest run && node unknown-after.js"}) + scope, err := DiscoverValidationScope(root, "vitest") + require.NoError(t, err) + require.Len(t, scope.Commands, 1) + require.NotEmpty(t, scope.Unresolved) + check := checkCIRuntimes(t.Context(), root, nil, nil, "vitest") + require.Equal(t, "incompatible", check.Status) + require.Len(t, check.Jobs, 2) + require.Contains(t, check.Jobs[0].Reason, "unknown-setup.js") + require.Equal(t, "missing_instrumentation", check.Jobs[1].Code) +} + +func TestScopeDirectoryChangesAndUnsafeInputs(t *testing.T) { + root := t.TempDir() + writeScripts(t, root, map[string]string{"test": "vitest"}) + writeScripts(t, filepath.Join(root, "app"), map[string]string{"test": "vitest run --coverage"}) + for _, tc := range []struct { + command string + count int + directory string + }{ + {"cd app && npm test", 1, "app"}, + {"cd app && cd .. && npm test", 1, "."}, + {"cd missing && npm test", 0, ""}, + {"cd /tmp && npm test", 0, ""}, + {"cd ../ && npm test", 0, ""}, + {"cd $(node setup.js) && npm test", 0, ""}, + } { + t.Run(tc.command, func(t *testing.T) { + remaining := 256 + result := resolveJestSequence(root, root, tc.command, nil, &remaining, "vitest") + require.Len(t, result.Tests, tc.count) + if tc.count > 0 { + require.Empty(t, result.Reason) + require.Equal(t, tc.directory, result.Tests[0].Directory) + } else { + require.NotEmpty(t, result.Reason) + } + }) + } + for _, command := range []string{"yarn --frozen-lockfile", "yarn --non-interactive --frozen-lockfile"} { + require.Empty(t, resolveJestCommand(root, command, nil).Reason) + } + require.NotEmpty(t, resolveJestCommand(root, "yarn --unknown-option", nil).Reason) + require.Equal(t, "CI=true vitest run", ValidationCommands(root, "CI=true npx vitest run", "vitest")[0].Command, "do not discard shell environment for batch execution") + require.Equal(t, "CI=true vitest", ValidationCommands(root, "CI=true npm test", "vitest")[0].Command, "preserve environment through package aliases") +} + +func TestDynamicSetupRetainsTestsButDoesNotValidateVariants(t *testing.T) { + root := newJestRepository(t, `jobs: + source: + steps: + - run: pnpm run test:types + - run: pnpm test + react: + steps: + - run: | + pnpm add -D react@$REACT_VER react-dom@$REACT_VER + pnpm test + built: + steps: + - run: sed -i~ 's/src/dist/' vitest.config.mts + - run: pnpm test +`) + writeScripts(t, root, map[string]string{"test": "vitest run", "test:types": "tsc --noEmit"}) + scope, err := DiscoverValidationScope(root, "vitest") + require.NoError(t, err) + require.Len(t, scope.Commands, 3, "source, built and alternate dependency tests must not collapse") + for _, entry := range scope.Commands { + require.Equal(t, "vitest run", entry.Command) + if !strings.Contains(entry.Locations[0], " / react /") { + require.Empty(t, entry.UnvalidatedSetup) + } else { + require.NotEmpty(t, entry.UnvalidatedSetup) + } + } + require.Contains(t, strings.Join(scope.Unresolved, "\n"), "react@$REACT_VER") + check := checkCIRuntimes(t.Context(), root, nil, nil, "vitest") + var missing []string + for _, job := range check.Jobs { + if job.Code == "missing_instrumentation" { + missing = append(missing, job.Job) + } + } + require.ElementsMatch(t, []string{"source", "react", "built"}, missing) +} + +func TestPartialShellDiscoveryPreservesStateBoundaries(t *testing.T) { + root := t.TempDir() + writeScripts(t, root, map[string]string{"test": "vitest run"}) + for _, command := range []string{ + "pnpm add react@$VERSION && pnpm test", + "pnpm add react@$(node version.js)\npnpm test", + "node setup.js > setup.log && pnpm test", + } { + remaining := 256 + result := resolveJestSequence(root, root, command, nil, &remaining, "vitest") + require.Len(t, result.Tests, 1, command) + require.NotEmpty(t, result.Tests[0].UnvalidatedSetup, command) + require.NotEmpty(t, result.Reason, command) + require.False(t, result.SingleJest, command) + } + for _, command := range []string{ + "cd $DIR && pnpm test", "source $SETUP && pnpm test", + "export NODE_OPTIONS=$OPTIONS; pnpm test", "eval $SETUP && pnpm test", + "if true; then cd app; fi; pnpm test", "pnpm add $DEP | pnpm test", + "pnpm add $DEP & pnpm test", "$COMMAND && pnpm test", + } { + remaining := 256 + result := resolveJestSequence(root, root, command, nil, &remaining, "vitest") + require.Empty(t, result.Tests, command) + require.NotEmpty(t, result.Reason, command) + } +} + +func TestPnpmNativeCommandsAreNotPackageScriptAliases(t *testing.T) { + root := t.TempDir() + writeScripts(t, root, map[string]string{"test": "vitest run"}) + for _, command := range []string{"pnpm publish --no-git-checks", "pnpm dlx pkg-pr-new publish './dist' --compact"} { + result := resolveJestCommand(root, command, nil) + require.Empty(t, result.Reason, command) + require.Contains(t, result.ReviewReason, "review separately", command) + } + remaining := 256 + result := resolveJestSequence(root, root, "pnpm add -D react@18 && pnpm test", nil, &remaining, "vitest") + require.Len(t, result.Tests, 1) + require.Empty(t, result.Reason) + require.Empty(t, result.Tests[0].UnvalidatedSetup) + require.Equal(t, "dependencies", result.Tests[0].Prerequisites[0].Kind) + writeScripts(t, root, map[string]string{"prepublishOnly": "vitest run"}) + require.Contains(t, resolveJestCommand(root, "pnpm publish --no-git-checks", nil).Reason, "may run tests") + writeScripts(t, root, map[string]string{"prepublish": "vitest run"}) + require.Contains(t, resolveJestCommand(root, "pnpm publish", nil).Reason, "prepublish") +} diff --git a/internal/onboard/scripts.go b/internal/onboard/scripts.go new file mode 100644 index 00000000..0a47b759 --- /dev/null +++ b/internal/onboard/scripts.go @@ -0,0 +1,516 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/kballard/go-shellquote" +) + +// A resolution can be unrelated, matched, or unresolved. A matched command with +// an unresolved component is still inconclusive; it must not hide custom wrappers. +type commandResolution struct { + Tests []TestCommand + Builds []BuildCommand + Matched bool + Benchmark bool + SingleJest bool + Evidence string + Reason string + Review bool + UnknownExecutable bool + ReviewReason string +} + +func unresolvedCommand(reason string) commandResolution { + return commandResolution{Reason: reason} +} + +func resolveTestStep(root string, workflow ciWorkflow, job runtimeJob, step runtimeStep, language, framework string) commandResolution { + if step.ResolutionError != "" { + return unresolvedCommand(step.ResolutionError) + } + if strings.TrimSpace(step.Run) == "" { + return commandResolution{} + } + if language != "javascript" || (framework != "jest" && framework != "vitest") { + return commandResolution{Matched: looksLikeTestJob(strings.ToLower(step.Run), language, framework)} + } + directory := stepDirectory(workflow, job, step) + shell := workflow.Defaults.Run.Shell + for _, defaults := range []runDefaults{job.Defaults.Run, {WorkingDirectory: step.WorkingDirectory, Shell: step.Shell}} { + if defaults.Shell != "" { + shell = defaults.Shell + } + } + if shell != "" && shell != "bash" && shell != "sh" { + return unresolvedCommand("Unsupported CI shell: " + shell) + } + if strings.ContainsAny(directory, "$`~") || (directory != "" && !filepath.IsLocal(directory)) { + return unresolvedCommand("Cannot statically resolve repository working-directory: " + directory) + } + if result, ok := resolveReleaseInput(filepath.Join(root, directory), step.Run, framework); ok { + return result + } + if jsonPackagingStep(step.Run) { + return commandResolution{Review: true, Reason: "JSON packaging transformation is outside " + framework + " validation; review separately if its output is used by tests."} + } + if strings.TrimSpace(step.Run) == "bash <(curl -s https://codecov.io/bash)" { + return commandResolution{Review: true, Reason: "Legacy Codecov upload is outside identified " + framework + " entry points; the downloaded script is not inspected or validated."} + } + remaining := 256 + result := resolveJestSequence(root, filepath.Join(root, directory), step.Run, nil, &remaining, framework) + if _, err := staticCommands(step.Run); err != nil { + if metadata, ok := resolveMetadataStep(root, workflow, job, step, framework); ok { + return metadata + } + } + if !result.Matched && result.Reason == "" && result.ReviewReason != "" { + result.Review, result.Reason = true, result.ReviewReason + } + if result.UnknownExecutable && !result.Matched && separateCIEntryPoint(workflow, job, step) { + result.Review = true + result.Reason = "Build, publishing, or documentation entry point was not identified as " + framework + "; review separately if it also runs tests. " + result.Reason + } + return result +} + +// Scope by explicit commands, never job/step names or comments. These unresolved +// entry points remain visible for review; this is not proof they cannot run tests. +// Instrumented steps and unknown test wrappers must still block validation. +func separateCIEntryPoint(workflow ciWorkflow, job runtimeJob, step runtimeStep) bool { + for _, env := range []map[string]string{workflow.Env, job.Env, step.Env} { + if env["NODE_OPTIONS"] != "" { + return false + } + } + commands, err := staticCommands(step.Run) + if err != nil || len(commands) != 1 { + return false + } + words := commands[0] + if len(words) == 2 && words[0] == "npx" && words[1] == "semantic-release" { + return true + } + if len(words) < 2 || !slices.Contains([]string{"npm", "yarn", "pnpm", "bun"}, words[0]) { + return false + } + args := words[1:] + if words[0] == "npm" { + _, args, _ = npmPrefix(args) + } + if len(args) == 0 { + return false + } + if args[0] == "run" || args[0] == "run-script" { + args = args[1:] + } else if words[0] == "npm" { + return false + } + if len(args) != 1 { + return false + } + if slices.Contains([]string{"test:perf", "perf", "benchmark"}, args[0]) { + return true + } + name, _, _ := strings.Cut(args[0], ":") + return slices.Contains([]string{"build", "build-storybook", "storybook", "docs", "release", "api-extractor", "bundlewatch"}, name) +} + +// Resolve only ordinary static commands and package script aliases. Never run +// a shell, load JavaScript, or rewrite the CI entry point during discovery. +func resolveJestCommand(directory, command string, stack []string) commandResolution { + remaining := 256 + return resolveJestSequence(directory, directory, command, stack, &remaining) +} + +func resolveJestSequence(root, directory, command string, stack []string, remaining *int, frameworks ...string) commandResolution { + if len(stack) >= 16 { + return unresolvedCommand("Package script nesting exceeds 16 levels") + } + commands, err := discoverCommands(command) + if err != nil { + return unresolvedCommand(err.Error()) + } + var result commandResolution + var setup []string + for _, fragment := range commands { + words := fragment.Words + if fragment.Reason != "" { + result.Reason = fragment.Reason + result.UnknownExecutable = false + continue + } + *remaining-- + if *remaining < 0 { + return unresolvedCommand("Package script expansion exceeds 256 commands") + } + if len(words) > 0 && words[0] == "cd" { + if len(words) != 2 { + result.Reason = "cd requires one static repository directory" + return result + } + next, err := repositoryDirectory(root, directory, words[1]) + if err != nil { + result.Reason = err.Error() + return result + } + directory = next + continue + } + if preparation, ok := localPreparation(root, directory, words); ok { + result.Builds = append(result.Builds, preparation) + continue + } + if mutatesCIFiles(shellquote.Join(words...)) { + setup = append(setup, "CI file or dependency changes require separate validation: "+shellquote.Join(words...)) + } + part := resolveJestWords(root, directory, words, stack, remaining, frameworks...) + for i := range part.Tests { + part.Tests[i].Prerequisites = append(slices.Clone(result.Builds), part.Tests[i].Prerequisites...) + part.Tests[i].UnvalidatedSetup = append(slices.Clone(setup), part.Tests[i].UnvalidatedSetup...) + if result.Reason != "" { + part.Tests[i].UnvalidatedSetup = append(part.Tests[i].UnvalidatedSetup, result.Reason) + } + } + result.Tests = append(result.Tests, part.Tests...) + result.Builds = append(result.Builds, part.Builds...) + result.SingleJest = len(commands) == 1 && part.SingleJest + if part.ReviewReason != "" { + result.ReviewReason = part.ReviewReason + } + if part.Reason != "" { + if result.Reason == "" { + result.Reason = part.Reason + result.UnknownExecutable = part.UnknownExecutable + } else { + result.UnknownExecutable = result.UnknownExecutable && part.UnknownExecutable + } + } + result.Matched = result.Matched || part.Matched + result.Benchmark = result.Benchmark || part.Benchmark + if part.Evidence != "" { + if result.Evidence != "" { + result.Evidence += "; " + } + result.Evidence += part.Evidence + } + } + if result.Matched && result.Benchmark && result.Reason == "" { + result.Reason = "This command combines ordinary tests and Vitest benchmarks; benchmark instrumentation remains unvalidated." + } + return result +} + +func resolveJestWords(root, directory string, words, stack []string, remaining *int, frameworks ...string) commandResolution { + framework := "jest" + if len(frameworks) > 0 { + framework = frameworks[0] + } + if len(words) == 0 { + return commandResolution{} + } + command := shellquote.Join(words...) + var assignments []string + for len(words) > 0 && strings.Contains(words[0], "=") { + key, _, _ := strings.Cut(words[0], "=") + if key == "NODE_OPTIONS" || key == "PATH" || strings.HasPrefix(key, "DD_") { + return unresolvedCommand("Command overrides instrumentation or executable selection: " + command) + } + assignments = append(assignments, words[0]) + words = words[1:] + } + if len(words) == 0 { + return unresolvedCommand("Standalone environment assignment requires review: " + command) + } + if literalFileOperation(words) { + return commandResolution{} + } + if words[0] == "npm" && len(words) > 2 && (words[1] == "-g" || words[1] == "--global") && slices.Contains([]string{"i", "install"}, words[2]) { + return commandResolution{} + } + if words[0] == "npm" { + prefix, args, err := npmPrefix(words[1:]) + if err != nil { + return unresolvedCommand(err.Error()) + } + if prefix != "" { + target, err := repositoryDirectory(root, directory, prefix) + if err != nil { + return unresolvedCommand(err.Error()) + } + result := resolveJestWords(root, target, append([]string{"npm"}, args...), stack, remaining, frameworks...) + if result.Evidence != "" { + result.Evidence = command + " (in " + target + ") -> " + result.Evidence + } + // Local flag forwarding from the repository root is not validated + // for a test runner in a different package directory. + original, _ := filepath.EvalSymlinks(directory) + result.SingleJest = result.SingleJest && target == original + return result + } + } + if (words[0] == "npx" || words[0] == "pnpx") || (len(words) > 1 && ((words[0] == "pnpm" || words[0] == "npm") && words[1] == "exec" || words[0] == "pnpm" && words[1] == "dlx")) { + if words[0] == "npx" || words[0] == "pnpx" { + words = words[1:] + } else { + words = words[2:] + } + if len(words) > 0 && words[0] == "--" { + words = words[1:] + } + if len(words) == 0 { + return unresolvedCommand("Missing executable: " + command) + } + } + if words[0] == framework || words[0] == "./node_modules/.bin/"+framework || words[0] == "node_modules/.bin/"+framework || words[0] == filepath.Join(directory, "node_modules", ".bin", framework) { + if framework == "vitest" && len(words) > 1 && words[1] == "bench" { + return commandResolution{Benchmark: true, ReviewReason: "Vitest benchmark mode is not validated by the test adapter; do not add instrumentation based on ordinary test validation: " + command} + } + canonicalRoot, _ := filepath.EvalSymlinks(root) + canonicalDirectory, _ := filepath.EvalSymlinks(directory) + relative, _ := filepath.Rel(canonicalRoot, canonicalDirectory) + return commandResolution{Matched: true, SingleJest: true, Evidence: command, + Tests: []TestCommand{{Directory: filepath.ToSlash(relative), Command: shellquote.Join(append(assignments, append([]string{framework}, words[1:]...)...)...)}}} + } + if slices.Equal(words, []string{"corepack", "enable"}) { + return commandResolution{} + } + switch words[0] { + case "echo", "printf", "true", "false", "eslint", "prettier", "oxlint", "oxfmt", "mkdir", "cp": + return commandResolution{} + case "unbuild", "rollup", "webpack", "tsup", "tsc", "tsgo": + if (words[0] == "tsc" || words[0] == "tsgo") && slices.Contains(words[1:], "--noEmit") { + return commandResolution{ReviewReason: "Type checking is outside " + framework + " validation: " + command} + } + canonicalRoot, _ := filepath.EvalSymlinks(root) + canonicalDirectory, _ := filepath.EvalSymlinks(directory) + relative, _ := filepath.Rel(canonicalRoot, canonicalDirectory) + build := BuildCommand{Directory: filepath.ToSlash(relative), Command: shellquote.Join(append(assignments, words...)...)} + result := commandResolution{Builds: []BuildCommand{build}} + if words[0] != "tsc" && words[0] != "tsgo" { + result.ReviewReason = "Build/tool command is outside " + framework + " validation; review its configuration separately if it also runs tests: " + command + } + return result + case "jest", "playwright", "vitest", "mocha", "cypress": + return commandResolution{ReviewReason: "Other test framework is outside " + framework + " validation: " + command} + case "flow", "bundlewatch", "api-extractor", "docusaurus", "documentation", "automd", "codecov": + return commandResolution{ReviewReason: "Build/tool command is outside " + framework + " validation; review its configuration separately if it also runs tests: " + command} + case "clean-publish": + return reviewReleaseLifecycle(directory, command, framework, []string{"prepublishOnly", "prepack", "prepare", "postpack", "publish", "postpublish"}) + case "pkg-pr-new": + if len(words) > 1 && words[1] == "publish" { + return reviewReleaseLifecycle(directory, command, framework, []string{"prepublishOnly", "prepack", "prepare", "postpack", "publish", "postpublish"}) + } + return unresolvedCommand("Unknown publication command: " + command) + case "git": + if gitReleaseOperation(words) { + return commandResolution{ReviewReason: "Git release operation is outside " + framework + " validation; review separately, including any Git hooks: " + command} + } + if slices.Equal(words, []string{"git", "diff", "--quiet", "--exit-code"}) { + return commandResolution{ReviewReason: "Git content comparison is outside " + framework + " validation; review separately, including configured diff helpers."} + } + return unresolvedCommand("Unsupported git command requires review: " + command) + case "npm", "yarn", "pnpm", "bun": + // Yarn without a subcommand installs dependencies. npm accepts global + // options before its install subcommand. Neither is a Jest entry point. + if yarnImplicitInstall(words) { + return commandResolution{} + } + if words[0] == "npm" && len(words) > 2 && (words[1] == "-g" || words[1] == "--global") && slices.Contains([]string{"i", "install"}, words[2]) { + return commandResolution{} + } + if len(words) < 2 { + return unresolvedCommand("Missing package-manager subcommand: " + command) + } + if slices.Contains([]string{"npm", "pnpm", "yarn"}, words[0]) && words[1] == "version" { + return reviewReleaseLifecycle(directory, command, framework, []string{"preversion", "version", "postversion"}) + } + if (words[0] == "npm" || words[0] == "pnpm") && words[1] == "publish" { + return resolveNpmPublish(directory, words, framework) + } + if slices.Equal(words, []string{"npm", "config", "get", "cache"}) { + return commandResolution{} + } + // These are setup/metadata commands, not explicit test entry points. + if slices.Contains([]string{"ci", "install", "i", "--version", "--help"}, words[1]) { + return commandResolution{} + } + if words[0] == "pnpm" && words[1] == "add" { + return unresolvedCommand("Dependency-changing setup requires separate validation: " + command) + } + args := words[1:] + if words[0] == "bun" && args[0] == "test" { + // bun test invokes Bun's own runner, not the package's Jest script. + return commandResolution{} + } + if args[0] == "run" || args[0] == "run-script" { + args = args[1:] + } else if words[0] == "npm" && args[0] != "test" && args[0] != "t" { + return unresolvedCommand("Unsupported npm subcommand: " + command) + } + if len(args) == 0 || strings.HasPrefix(args[0], "-") { + return unresolvedCommand("Unsupported package script selection: " + command) + } + name := args[0] + if words[0] == "npm" && words[1] == "t" { + name = "test" + } + forwarded := args[1:] + if len(forwarded) > 0 && forwarded[0] == "--" { + forwarded = forwarded[1:] + } else if words[0] == "npm" && len(forwarded) > 0 { + return unresolvedCommand("npm script arguments without -- require review: " + command) + } + result := resolvePackageScript(root, directory, name, forwarded, stack, command, remaining, frameworks...) + for i := range result.Tests { + if len(assignments) > 0 { + result.Tests[i].Command = shellquote.Join(assignments...) + " " + result.Tests[i].Command + for j := range result.Tests[i].Prerequisites { + result.Tests[i].Prerequisites[j].Command = shellquote.Join(assignments...) + " " + result.Tests[i].Prerequisites[j].Command + } + } + } + for i := range result.Builds { + if len(assignments) > 0 { + result.Builds[i].Command = shellquote.Join(assignments...) + " " + result.Builds[i].Command + } + } + return result + default: + return commandResolution{Reason: "Cannot statically resolve command: " + command, UnknownExecutable: true} + } +} + +func resolvePackageScript(root, directory, name string, args, stack []string, command string, remaining *int, frameworks ...string) commandResolution { + key := filepath.Join(directory, "package.json") + ":" + name + if slices.Contains(stack, key) { + return unresolvedCommand("Package script cycle: " + strings.Join(append(slices.Clone(stack), key), " -> ")) + } + data, err := os.ReadFile(filepath.Join(directory, "package.json")) + if err != nil { + return unresolvedCommand("Cannot read package.json for " + command + ": " + err.Error()) + } + var manifest struct { + Scripts map[string]string `json:"scripts"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return unresolvedCommand("Cannot parse package.json: " + err.Error()) + } + script, ok := manifest.Scripts[name] + if !ok { + if name == "flow" && (strings.HasPrefix(command, "yarn flow") || strings.HasPrefix(command, "pnpm flow")) { + return resolveJestWords(root, directory, append([]string{"flow"}, args...), stack, remaining, frameworks...) + } + if len(frameworks) > 0 && frameworks[0] == "vitest" && name == "vitest" && (strings.HasPrefix(command, "pnpm vitest") || strings.HasPrefix(command, "yarn vitest")) { + return resolveJestWords(root, directory, append([]string{"vitest"}, args...), stack, remaining, frameworks...) + } + return unresolvedCommand("No package.json script named " + name + " in " + directory) + } + var lifecycle string + for _, hook := range []string{"pre" + name, "post" + name} { + if manifest.Scripts[hook] != "" { + lifecycle = "Package lifecycle script " + hook + " requires review alongside " + command + } + } + if len(args) > 0 { + script += " " + shellquote.Join(args...) + } + result := resolveJestSequence(root, directory, script, append(slices.Clone(stack), key), remaining, frameworks...) + // Preserve the package manager's build entry point, including its lifecycle + // and orchestration, when expansion cannot represent that build alone. + if name == "build" && !result.Matched && orchestratedBuild(root, directory, script, manifest.Scripts, stack, remaining, frameworks...) { + relative, _ := filepath.Rel(root, directory) + return commandResolution{Builds: []BuildCommand{{Directory: filepath.ToSlash(relative), Command: command, Kind: "package-build"}}} + } + if lifecycle != "" { + result.Reason = lifecycle + result.UnknownExecutable = false + } + if result.Reason != "" { + result.Reason = command + " -> " + result.Reason + } else if result.Matched { + result.Evidence = command + " -> " + result.Evidence + } + return result +} + +// Split a deliberately small shell subset: literal words, quotes, comments, +// newlines and &&/; sequences. Expansion, pipelines and control flow require +// review. shellquote handles word quoting; this scanner only finds boundaries. +func staticCommands(command string) ([][]string, error) { + var commands [][]string + start := 0 + var quote byte + add := func(end int) error { + words, err := shellquote.Split(command[start:end]) + if err != nil { + return err + } + if len(words) > 0 { + commands = append(commands, words) + } + if len(commands) > 64 { + return fmt.Errorf("command sequence exceeds 64 entries") + } + return nil + } + for i := 0; i < len(command); i++ { + c := command[i] + if c == '\\' && quote != '\'' { + i++ + continue + } + if quote != 0 { + if c == quote { + quote = 0 + } else if quote == '"' && (c == '$' || c == '`') { + return nil, fmt.Errorf("dynamic shell expansion requires review") + } + continue + } + switch c { + case '\'', '"': + quote = c + case '$', '`', '|', '<', '>', '(', ')', '{', '}': + return nil, fmt.Errorf("unsupported or dynamic shell syntax requires review") + case '#': + if i > start && !strings.ContainsRune(" \t\n", rune(command[i-1])) { + continue + } + if err := add(i); err != nil { + return nil, err + } + for i < len(command) && command[i] != '\n' { + i++ + } + start = i + 1 + case '&', ';', '\n': + if err := add(i); err != nil { + return nil, err + } + if c == '&' { + if i+1 >= len(command) || command[i+1] != '&' { + return nil, fmt.Errorf("background commands require review") + } + i++ + } + start = i + 1 + } + } + if start < len(command) { + if err := add(len(command)); err != nil { + return nil, err + } + } + return commands, nil +} diff --git a/internal/onboard/scripts_files.go b/internal/onboard/scripts_files.go new file mode 100644 index 00000000..bc435ab9 --- /dev/null +++ b/internal/onboard/scripts_files.go @@ -0,0 +1,69 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "path/filepath" + "strings" +) + +// These literal file operations are not test entry points. Flags that can load +// programs and sed's command-execution forms remain unresolved. Never execute IO. +func literalFileOperation(words []string) bool { + localPaths := func(paths []string) bool { + if len(paths) == 0 { + return false + } + for _, path := range paths { + if !filepath.IsLocal(path) || strings.HasPrefix(path, "-") || strings.ContainsAny(path, "*?[]$`\n:") { + return false + } + } + return true + } + switch words[0] { + case "rimraf", "rm": + return localPaths(words[1:]) + case "rsync": + return len(words) == 4 && words[1] == "-a" && localPaths(words[2:]) + case "sed": + args := words[1:] + if len(args) > 0 && strings.HasPrefix(args[0], "-i") { + args = args[1:] + } + if len(args) != 2 || !localPaths(args[1:]) { + return false + } + return literalSedSubstitution(args[0]) + } + return false +} + +// Accept a single substitution with no execution flag or following command. +func literalSedSubstitution(script string) bool { + if len(script) < 4 || len(script) > 4096 || script[0] != 's' || strings.ContainsAny(script, "\n\r") { + return false + } + delimiter := script[1] + if delimiter != '/' && delimiter != '#' && delimiter != '|' { + return false + } + separators := 0 + for i := 2; i < len(script); i++ { + if script[i] == '\\' { + i++ + continue + } + if script[i] == delimiter { + separators++ + if separators == 2 { + flags := script[i+1:] + return flags == "" || flags == "g" + } + } + } + return false +} diff --git a/internal/onboard/scripts_files_test.go b/internal/onboard/scripts_files_test.go new file mode 100644 index 00000000..5d997364 --- /dev/null +++ b/internal/onboard/scripts_files_test.go @@ -0,0 +1,71 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLiteralFileOperationsDoNotHideUnknownPrograms(t *testing.T) { + root := t.TempDir() + for _, command := range []string{ + "rimraf ./dist", "CI=true rimraf ./dist", "rm build/index.html.bak", "rsync -a docs/ build/docs", + "sed -i.bak 's###g' build/index.html", + `sed -i.bak 's/<\/body>/\n", "", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + run := preparedTestdrive(t) + run.framework = &framework.Mocha{} + installer := &fakeTracer{preloadPath: "/trace/ci/init.js"} + run.platform = installer + run.executor = &fakeTestdriveExecutor{output: []byte(tc.output), err: errors.New("exit status 1")} + run.startIntake = func(string, intake.Scenario) (localIntake, error) { + return &fakeIntake{findings: intake.Facts{TestEventCount: tc.events}}, nil + } + var output bytes.Buffer + require.Error(t, run.Run(t.Context(), &output), "Mocha remains unvalidated") + require.Contains(t, output.String(), "Mocha command output:") + require.Contains(t, output.String(), tc.want) + require.Contains(t, output.String(), "Bounded diagnostic: "+validationPath("")) + require.NotContains(t, output.String(), "Full test output:") + require.NotContains(t, output.String(), "see the full test output") + if tc.name == "long" { + require.Contains(t, output.String(), "initial failure") + require.Contains(t, output.String(), "final failure") + require.Equal(t, 78, strings.Count(output.String(), "log line")) + } + require.NoDirExists(t, installer.sessionDirectory) + report := readValidationReport(t, run.repositoryRoot) + require.Len(t, report.Runs, 1) + require.Equal(t, -1, *report.Runs[0].ExitCode, "generic executor errors have no process exit code") + require.Equal(t, tc.events, report.Runs[0].TestEventCount) + require.LessOrEqual(t, len([]rune(report.Runs[0].Diagnostic)), 1024) + page, err := os.ReadFile(htmlReportPath(run.repositoryRoot)) + require.NoError(t, err) + require.Contains(t, html.UnescapeString(string(page)), tc.output) + require.Contains(t, html.UnescapeString(string(page)), report.Runs[0].Command) + require.Contains(t, string(page), "Command failed: exit status 1") + require.NotContains(t, string(page), "") + if tc.output == "" { + require.Contains(t, string(page), "The command produced no output.") + } + if tc.output != "" { + require.NotEmpty(t, report.Runs[0].Diagnostic) + } + files, err := os.ReadDir(filepath.Dir(validationPath(run.repositoryRoot))) + require.NoError(t, err) + require.Len(t, files, 2, "retain HTML and JSON, without raw output files") + }) + } +} + +func TestReportReplacementKeepsOneFileAndPreservesOtherProjectData(t *testing.T) { + root := t.TempDir() + directory := filepath.Dir(validationPath(root)) + require.NoError(t, os.MkdirAll(directory, 0755)) + requireWriteFile(t, filepath.Join(directory, "customer-plan.json"), "original plan") + for _, name := range []string{"first", "second"} { + result := validationResult{Session: name, Compatibility: verdict{Status: "compatible"}} + require.NoError(t, finishValidation(&bytes.Buffer{}, root, result)) + } + data, err := os.ReadFile(validationPath(root)) + require.NoError(t, err) + var result validationResult + require.NoError(t, json.Unmarshal(data, &result)) + require.Equal(t, "second", result.Session) + files, err := os.ReadDir(directory) + require.NoError(t, err) + require.Len(t, files, 2) + data, err = os.ReadFile(filepath.Join(directory, "customer-plan.json")) + require.NoError(t, err) + require.Equal(t, "original plan", string(data)) +} + +func TestAtomicReportFailureRemovesOnlyItsScratchFile(t *testing.T) { + root := t.TempDir() + target := validationPath(root) + require.NoError(t, os.MkdirAll(target, 0755)) + requireWriteFile(t, filepath.Join(target, "customer-file"), "keep") + require.Error(t, finishValidation(&bytes.Buffer{}, root, validationResult{})) + files, err := os.ReadDir(filepath.Dir(target)) + require.NoError(t, err) + require.Len(t, files, 1) + require.FileExists(t, filepath.Join(target, "customer-file")) +} + +func TestConcurrentReportWritesLeaveOneCompleteReport(t *testing.T) { + root := t.TempDir() + var group sync.WaitGroup + failures := make(chan error, 8) + for range 8 { + group.Go(func() { + failures <- finishValidation(&bytes.Buffer{}, root, validationResult{Compatibility: verdict{Status: "compatible"}}) + }) + } + group.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } + data, err := os.ReadFile(validationPath(root)) + require.NoError(t, err) + require.True(t, json.Valid(data)) + files, err := os.ReadDir(filepath.Dir(validationPath(root))) + require.NoError(t, err) + require.Len(t, files, 1) +} + +func TestProbeRemovedWhenScenarioSetupFails(t *testing.T) { + run := preparedTestdrive(t) + source := filepath.Join(run.repositoryRoot, "existing.test.js") + requireWriteFile(t, source, "original test") + run.executor = discoveryExecutor(func(args []string) ([]byte, error) { + path := source + for i, arg := range args { + if arg == "--runTestsByPath" { + path = args[i+1] + } + } + return json.Marshal([]string{path}) + }) + session, err := NewSession() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, session.Close()) }) + run.startIntake = func(string, intake.Scenario) (localIntake, error) { return nil, errors.New("listener failed") } + result := validationResult{Preflight: &jestPreflight{Projects: []jestProject{{Root: run.repositoryRoot}}}} + require.ErrorContains(t, run.runJestFeatures(t.Context(), &bytes.Buffer{}, session, "/trace/ci/init.js", &result), "listener failed") + probes, err := filepath.Glob(filepath.Join(run.repositoryRoot, "ddtest*")) + require.NoError(t, err) + require.Empty(t, probes) + data, err := os.ReadFile(source) + require.NoError(t, err) + require.Equal(t, "original test", string(data)) +} diff --git a/internal/testdrive/command_selection_test.go b/internal/testdrive/command_selection_test.go new file mode 100644 index 00000000..6bf8b6dd --- /dev/null +++ b/internal/testdrive/command_selection_test.go @@ -0,0 +1,107 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/DataDog/ddtest/internal/settings" + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/stretchr/testify/require" +) + +func TestPrepareUsesCICommandWithoutLoadingJestConfig(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + previous := settings.Get().Command + settings.Get().Command = "" + t.Cleanup(func() { settings.Get().Command = previous }) + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"jest","test:ci":"jest --runInBand --no-cache --coverage --verbose"},"devDependencies":{"jest":"29.7.0"}}`) + require.NoError(t, os.MkdirAll(filepath.Join(root, ".github/workflows"), 0755)) + requireWriteFile(t, filepath.Join(root, ".github/workflows/test.yml"), "jobs:\n tests:\n steps:\n - run: npm run test:ci\n") + requireWriteFile(t, filepath.Join(root, "jest.config.js"), "throw new Error('prepare must not execute config')") + drive, err := Prepare("latest") + require.NoError(t, err) + require.Equal(t, "npm", drive.command) + require.Equal(t, []string{"run", "test:ci"}, drive.args) + var output bytes.Buffer + drive.Preview(&output) + require.Contains(t, output.String(), "run: npm run test:ci") + settings.Get().Command = "node_modules/.bin/jest --config explicit.js" + drive, err = Prepare("latest") + require.NoError(t, err) + require.Equal(t, "node_modules/.bin/jest", drive.command) + require.Equal(t, []string{"--config", "explicit.js"}, drive.args) +} + +func TestJestCoverageOutputUsesSessionDirectory(t *testing.T) { + drive := preparedTestdrive(t) + executor := &fakeTestdriveExecutor{} + drive.executor = executor + drive.startIntake = func(string, intake.Scenario) (localIntake, error) { return &fakeIntake{url: "http://127.0.0.1:1"}, nil } + session, err := NewSession() + require.NoError(t, err) + defer func() { require.NoError(t, session.Close()) }() + for _, probe := range []string{"", "probe.test.js"} { + _, err := drive.runJest(t.Context(), &bytes.Buffer{}, session, "/tracer/ci/init.js", "baseline", false, intake.Scenario{}, probe, "pass") + require.NoError(t, err) + index := slices.Index(executor.args, "--coverageDirectory") + require.NotEqual(t, -1, index) + require.Equal(t, filepath.Join(session.Directory(), "baseline", "coverage"), executor.args[index+1]) + } +} + +func TestJestValidationPreservesCommandSeparators(t *testing.T) { + for _, tc := range []struct { + name, command string + args, prefix []string + }{ + {"direct", "jest", []string{"--runInBand", "--", "original.test.js"}, []string{"--runInBand"}}, + {"node", "node", []string{"node_modules/jest/bin/jest.js", "--", "original.test.js"}, []string{"node_modules/jest/bin/jest.js"}}, + {"npx wrapper", "npx", []string{"--", "jest", "--", "original.test.js"}, []string{"--", "jest"}}, + {"npm script", "npm", []string{"test", "--", "--", "original.test.js"}, []string{"test", "--"}}, + } { + t.Run(tc.name, func(t *testing.T) { + original := slices.Clone(tc.args) + want := append(slices.Clone(tc.prefix), "--json", "--", "original.test.js") + require.Equal(t, want, appendJestArgs(tc.command, tc.args, "--json")) + run := preparedTestdrive(t) + run.command, run.args = tc.command, tc.args + executor := &fakeTestdriveExecutor{output: []byte(`["probe.test.js"]`)} + run.executor = executor + files, _, err := run.discoverJestTests(t.Context(), "probe.test.js") + require.NoError(t, err) + require.Equal(t, []string{"probe.test.js"}, files) + want = append(slices.Clone(tc.prefix), "--listTests", "--json", "--runTestsByPath", "probe.test.js", "--") + require.Equal(t, want, executor.args) + run.startIntake = func(string, intake.Scenario) (localIntake, error) { return &fakeIntake{url: "http://127.0.0.1:1"}, nil } + session, err := NewSession() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, session.Close()) }) + _, err = run.runJest(t.Context(), &bytes.Buffer{}, session, "/tracer/ci/init.js", "probe", true, intake.Scenario{}, "probe.test.js", "pass") + require.NoError(t, err) + require.NotContains(t, executor.args, "original.test.js") + require.Equal(t, "--", executor.args[len(executor.args)-1], "generated options must precede Jest's separator") + require.Contains(t, executor.args, "--coverageThreshold={}") + require.Contains(t, executor.args, "--testNamePattern") + require.Equal(t, original, tc.args, "probe commands must not mutate the compatibility command") + selected, ok := projectArgs(tc.command, tc.args, "Web") + require.True(t, ok) + want = append(slices.Clone(tc.prefix), "--selectProjects", "Web", "--", "original.test.js") + require.Equal(t, want, selected) + }) + } +} + +func TestJestValidationPreservesPackageManagerForwarding(t *testing.T) { + require.Equal(t, []string{"test", "--", "--json"}, appendJestArgs("npm", []string{"test"}, "--json")) + require.Equal(t, []string{"test", "--", "--runInBand", "--json"}, appendJestArgs("npm", []string{"test", "--", "--runInBand"}, "--json")) + require.Equal(t, []string{"--", "jest", "--json"}, appendJestArgs("npx", []string{"--", "jest"}, "--json")) +} diff --git a/internal/testdrive/configuration_workspace.go b/internal/testdrive/configuration_workspace.go new file mode 100644 index 00000000..87c615ef --- /dev/null +++ b/internal/testdrive/configuration_workspace.go @@ -0,0 +1,131 @@ +package testdrive + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "maps" + "os" + "path/filepath" +) + +type configurationExecutor struct { + commandExecutor + environment map[string]string +} + +func (e configurationExecutor) CombinedOutput(ctx context.Context, name string, args []string, env map[string]string) ([]byte, error) { + merged := map[string]string{} + maps.Copy(merged, e.environment) + maps.Copy(merged, env) + return e.commandExecutor.CombinedOutput(ctx, name, args, merged) +} + +// Commands still use the existing process executor. The temporary copy gives +// every variant independent manifests, dependencies, test files and build output. +func (t *Testdrive) runPreparedConfiguration(ctx context.Context, output io.Writer, item *configurationResult) (result validationResult, runErr error) { + if len(item.Prerequisites) == 0 || t.checkOnly { + if err := t.runBuildPrerequisites(ctx, output, item); err != nil { + return result, err + } + t.executor = configurationExecutor{t.executor, item.Environment} + return t.run(ctx, output) + } + original := t.repositoryRoot + workspace, err := os.MkdirTemp("", "ddtest-configuration-") + if err != nil { + return result, err + } + defer func() { + err := os.RemoveAll(workspace) + if err != nil { + result.Cleanup = &verdict{Status: "failed", Reason: "Could not remove configuration workspace: " + workspace} + } + runErr = errors.Join(runErr, err) + }() + if err := copyConfiguration(ctx, original, workspace); err != nil { + return result, err + } + cwd, err := os.Getwd() + if err != nil { + return result, err + } + if err := os.Chdir(workspace); err != nil { + return result, err + } + defer func() { runErr = errors.Join(runErr, os.Chdir(cwd)) }() + t.command = filepath.Join(workspace, "node_modules", ".bin", filepath.Base(t.command)) + t.repositoryRoot = workspace + if err := t.runBuildPrerequisites(ctx, output, item); err != nil { + result.Compatibility = verdict{Status: "not exercised", Reason: "Configuration preparation failed; tests were not started."} + return result, err + } + t.executor = configurationExecutor{t.executor, item.Environment} + return t.run(ctx, output) +} + +func copyConfiguration(ctx context.Context, source, target string) error { + canonical, err := filepath.EvalSymlinks(source) + if err != nil { + return err + } + source = canonical + return filepath.WalkDir(source, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + relative, err := filepath.Rel(source, path) + if err != nil { + return err + } + if relative == ".testoptimization" { + return filepath.SkipDir + } + destination := filepath.Join(target, relative) + info, err := entry.Info() + if err != nil { + return err + } + if entry.IsDir() { + return os.MkdirAll(destination, info.Mode().Perm()|0700) + } + if info.Mode()&os.ModeSymlink != 0 { + link, err := os.Readlink(path) + if err != nil { + return err + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return fmt.Errorf("cannot isolate symlink %s: %w", relative, err) + } + local, err := filepath.Rel(source, resolved) + if err != nil || !filepath.IsLocal(local) { + return fmt.Errorf("configuration symlink escapes repository: %s", relative) + } + if filepath.IsAbs(link) { + link = filepath.Join(target, local) + } + return os.Symlink(link, destination) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("cannot isolate special file %s", relative) + } + // Git objects can be read-only; the copy must never share writable inodes. + in, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm()|0600) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + return errors.Join(copyErr, out.Close()) + }) +} diff --git a/internal/testdrive/configuration_workspace_test.go b/internal/testdrive/configuration_workspace_test.go new file mode 100644 index 00000000..c017589c --- /dev/null +++ b/internal/testdrive/configuration_workspace_test.go @@ -0,0 +1,34 @@ +package testdrive + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfigurationCopyIsIndependentAndRejectsEscapingLinks(t *testing.T) { + source, target := t.TempDir(), t.TempDir() + requireWriteFile(t, filepath.Join(source, "package.json"), "original") + require.NoError(t, os.MkdirAll(filepath.Join(source, "node_modules/pkg"), 0755)) + requireWriteFile(t, filepath.Join(source, "node_modules/pkg/index.js"), "original dependency") + require.NoError(t, os.Symlink("pkg", filepath.Join(source, "node_modules/link"))) + require.NoError(t, os.Mkdir(filepath.Join(source, ".testoptimization"), 0755)) + requireWriteFile(t, filepath.Join(source, ".testoptimization/testdrive.json"), "old evidence") + require.NoError(t, copyConfiguration(t.Context(), source, target)) + requireWriteFile(t, filepath.Join(target, "package.json"), "changed") + requireWriteFile(t, filepath.Join(target, "node_modules/link/index.js"), "changed dependency") + for path, expected := range map[string]string{"package.json": "original", "node_modules/pkg/index.js": "original dependency"} { + data, err := os.ReadFile(filepath.Join(source, path)) + require.NoError(t, err) + require.Equal(t, expected, string(data)) + } + require.NoFileExists(t, filepath.Join(target, ".testoptimization/testdrive.json")) + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(source, "outside"))) + require.ErrorContains(t, copyConfiguration(t.Context(), source, t.TempDir()), "escapes repository") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, copyConfiguration(ctx, source, t.TempDir()), context.Canceled) +} diff --git a/internal/testdrive/configurations.go b/internal/testdrive/configurations.go new file mode 100644 index 00000000..3f450e73 --- /dev/null +++ b/internal/testdrive/configurations.go @@ -0,0 +1,267 @@ +package testdrive + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "path/filepath" + "slices" + "strings" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/kballard/go-shellquote" +) + +type configurationResult struct { + onboard.TestCommand + Status string `json:"status"` + Compatibility verdict `json:"compatibility"` + Preflight *jestPreflight `json:"preflight,omitempty"` + Error string `json:"error,omitempty"` + Preparation []buildResult `json:"preparation,omitempty"` +} + +func setSingleConfigurationScope(root string, result *validationResult, scope onboard.ValidationScope) { + result.Scope = &scope + var selected []onboard.TestCommand + if result.Preflight != nil { + selected = onboard.ValidationCommands(root, result.Preflight.Command, result.Framework) + } + for _, entry := range scope.Commands { + item := configurationResult{TestCommand: entry, Status: "not exercised", Compatibility: verdict{Status: "not exercised", Reason: "Run testdrive --all after required setup to validate every discovered configuration."}} + for _, command := range selected { + if command.Command == entry.Command && command.Directory == entry.Directory && slices.EqualFunc(command.Prerequisites, entry.Prerequisites, onboard.EqualPreparation) && len(entry.UnvalidatedSetup) == 0 && maps.Equal(command.Environment, entry.Environment) { + item.Compatibility = result.Compatibility + item.Preflight = result.Preflight + item.Error = result.Error + item.Status = configurationStatus(*result) + } + } + result.Configurations = append(result.Configurations, item) + } +} + +func configurationStatus(result validationResult) string { + if result.CheckOnly { + return "not exercised" + } + if result.Error != "" || result.Compatibility.Status != "compatible" || len(result.Features) == 0 { + return "incomplete" + } + for _, feature := range result.Features { + if feature.Status != "passed" { + return "incomplete" + } + } + return "passed" +} + +func applyScopeVerdict(result *validationResult) { + if result.Scope == nil { + return + } + if len(result.Scope.Unresolved) > 0 { + result.Success = false + result.ChecksPassed = false + } + for _, configuration := range result.Configurations { + if configuration.Status != "passed" { + result.Success = false + result.LocalSuccess = false + } + } +} + +// All evidence comes from this invocation. No earlier success is reused after +// source, configuration, runtime or tracer edits. Each configuration has its own +// paired comparison and feature controls, and shares only the final report pair. +func (t *Testdrive) runAllConfigurations(ctx context.Context, output io.Writer) error { + scope, err := onboard.DiscoverValidationScope(t.repositoryRoot, t.framework.Name()) + if err != nil { + return err + } + if len(scope.Commands) == 0 { + return fmt.Errorf("no explicit %s CI invocations discovered; review discovery and use --command for local validation", t.framework.Name()) + } + result := validationResult{Session: planSession().ID(), Framework: t.framework.Name(), Tracer: t.tracerLabel, CheckOnly: t.checkOnly, Scope: &scope, + Compatibility: verdict{Status: "compatible", Reason: "Every discovered configuration passed its own paired comparison; configurations were not compared against each other."}, Cleanup: &verdict{Status: "passed", Reason: "Temporary sessions removed for all configurations."}} + var models []reportModel + var runErrors []error + for _, entry := range scope.Commands { + item := configurationResult{TestCommand: entry, Status: "not exercised", Compatibility: verdict{Status: "not exercised", Reason: "Configuration did not execute."}} + words, parseErr := shellquote.Split(entry.Command) + if filepath.Clean(entry.Directory) != "." { + parseErr = fmt.Errorf("configuration in %s needs validation from that package directory; --all currently executes root-package commands only", entry.Directory) + } + if len(words) == 0 || strings.Contains(words[0], "=") { + parseErr = fmt.Errorf("configuration command needs explicit environment/wrapper review: %s", entry.Command) + } + if len(entry.UnvalidatedSetup) > 0 { + parseErr = fmt.Errorf("configuration requires separate CI setup; paired execution was not started: %s", strings.Join(entry.UnvalidatedSetup, "; ")) + } + if ctx.Err() != nil { + parseErr = ctx.Err() + } + if parseErr != nil { + item.Error = reportText(parseErr.Error()) + result.Compatibility = verdict{Status: "inconclusive", Reason: "At least one discovered configuration did not execute; see configurations."} + runErrors = append(runErrors, parseErr) + result.Configurations = append(result.Configurations, item) + continue + } + _, _ = fmt.Fprintf(output, "\nConfiguration %d/%d: %s\n", len(result.Configurations)+1, len(scope.Commands), entry.Command) + + label := fmt.Sprintf("configuration %d: %s", len(result.Configurations)+1, entry.Command) + for _, key := range slices.Sorted(maps.Keys(entry.Environment)) { + label += " (" + key + "=" + entry.Environment[key] + ")" + } + child := *t + child.allConfigurations = false + child.session = planSession() + if result.Selection != nil && result.Selection.Version != "" { + child.tracerVersion = result.Selection.Version + } + child.command = filepath.Join(t.repositoryRoot, "node_modules", ".bin", words[0]) + child.args = words[1:] + child.reportModels = &models + // Rebind method values to this command, rather than the original drive. + child.preflight = child.checkJestPreflight + if t.framework.Name() == "vitest" { + child.preflight = child.checkVitestPreflight + } + before := len(models) + execution, childErr := child.runPreparedConfiguration(ctx, output, &item) + prepareValidationResult(t.repositoryRoot, &execution) + item.Status = configurationStatus(execution) + if execution.Preflight == nil && childErr != nil { + item.Status = "blocked" + } + item.Compatibility = execution.Compatibility + item.Preflight = execution.Preflight + item.Error = execution.Error + if childErr != nil && item.Error == "" { + item.Error = reportText(childErr.Error()) + } + result.Configurations = append(result.Configurations, item) + if childErr != nil { + runErrors = append(runErrors, fmt.Errorf("%s: %w", entry.Command, childErr)) + } + if execution.Compatibility.Status != "compatible" { + result.Compatibility = verdict{Status: "inconclusive", Reason: "At least one discovered configuration did not pass its paired comparison; see configurations."} + } + if result.Preflight == nil { + result.Preflight = execution.Preflight + } + if result.Selection == nil { + result.Selection = execution.Selection + result.Tracer = execution.Tracer + result.TracerSource = execution.TracerSource + } + if result.Tracer != execution.Tracer { + runErrors = append(runErrors, fmt.Errorf("tracer selection changed between configurations; select an exact --tracer-version")) + } + result.CIRuntime = execution.CIRuntime + if result.CISelection == nil || execution.CISelection != nil && execution.CISelection.Status != "compatible" { + result.CISelection = execution.CISelection + } + if execution.Cleanup != nil && execution.Cleanup.Status != "passed" { + result.Cleanup = execution.Cleanup + } + for _, run := range execution.Runs { + run.Configuration = label + result.Runs = append(result.Runs, run) + } + for _, feature := range execution.Features { + feature.Project = strings.TrimSuffix(label+" / "+feature.Project, " / ") + result.Features = append(result.Features, feature) + } + if len(models) > before+1 { + models[before] = models[len(models)-1] + models = models[:before+1] + } + for i := before; i < len(models); i++ { + labelConfigurationModel(&models[i], label) + } + } + if t.checkOnly { + result.Compatibility = verdict{Status: "not exercised", Reason: "Configuration checks only; run --all without --check-only for all paired executions and features."} + } + if len(models) > 0 { + if err := writeConfigurationHTML(t.repositoryRoot, models); err != nil { + runErrors = append(runErrors, err) + } else { + result.HTMLReportCurrent = true + } + } + runErr := errors.Join(runErrors...) + if runErr != nil { + result.Error = runErr.Error() + } + return errors.Join(runErr, finishValidation(output, t.repositoryRoot, result)) +} + +func labelConfigurationModel(model *reportModel, command string) { + for i := range model.Tests { + model.Tests[i].Label = command + " › " + model.Tests[i].Label + } + for i := range model.Suites { + model.Suites[i].Name = command + " › " + model.Suites[i].Name + } + for i := range model.Cards { + model.Cards[i].Title = command + " — " + model.Cards[i].Title + } +} + +func writeConfigurationHTML(root string, models []reportModel) error { + model := reportModel{NoTestEvents: true, Summary: fmt.Sprintf("%d configurations reported. See validation JSON for completeness, compatibility and feature verdicts.", len(models)), Artifacts: []reportArtifact{{Title: "Validation JSON", Href: validationFilename}}} + var commands, outputs, failures []string + for index, part := range models { + if index == 0 { + model.Runtime = reportRuntime{Framework: part.Runtime.Framework, Tracer: part.Runtime.Tracer} + model.CoverageLevel = part.CoverageLevel + } else if model.CoverageLevel != part.CoverageLevel { + model.CoverageLevel = "" + } + model.NoTestEvents = model.NoTestEvents && part.NoTestEvents + label := fmt.Sprintf("Configuration %d: %s", index+1, part.Runtime.Command) + commands = append(commands, label) + outputs = append(outputs, label+"\n"+part.Runtime.Output) + if part.Runtime.Error != "" { + failures = append(failures, label+": "+part.Runtime.Error) + } + // Suite links address positions in the combined test table. Clone the + // rows because finding cards can share their source model's slices. + offset := len(model.Tests) + part.Suites = offsetSuiteTests(part.Suites, offset) + part.Cards = slices.Clone(part.Cards) + for i := range part.Cards { + part.Cards[i].Suites = offsetSuiteTests(part.Cards[i].Suites, offset) + } + model.Cards = append(model.Cards, part.Cards...) + model.Suites = append(model.Suites, part.Suites...) + model.Tests = append(model.Tests, part.Tests...) + } + model.Runtime.Command = strings.Join(commands, "\n") + model.Runtime.Output = strings.Join(outputs, "\n\n") + model.Runtime.Error = strings.Join(failures, "\n") + model.Facts = []reportFact{{Label: "Configurations reported", Value: fmt.Sprint(len(models))}, {Label: "Tests", Value: fmt.Sprint(len(model.Tests))}} + var content bytes.Buffer + if err := testdriveReport.Execute(&content, model); err != nil { + return err + } + return writeValidation(htmlReportPath(root), content.Bytes()) +} + +func offsetSuiteTests(suites []reportSuite, offset int) []reportSuite { + suites = slices.Clone(suites) + for i := range suites { + suites[i].Tests = slices.Clone(suites[i].Tests) + for j := range suites[i].Tests { + suites[i].Tests[j].TestIndex += offset + } + } + return suites +} diff --git a/internal/testdrive/configurations_test.go b/internal/testdrive/configurations_test.go new file mode 100644 index 00000000..08a56b73 --- /dev/null +++ b/internal/testdrive/configurations_test.go @@ -0,0 +1,140 @@ +package testdrive + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/stretchr/testify/require" +) + +func TestSelectedCommandDoesNotCertifyOtherConfigurations(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"scripts":{"test":"vitest run"}}`), 0644)) + scope := onboard.ValidationScope{Commands: []onboard.TestCommand{{Directory: ".", Command: "vitest run"}, {Directory: ".", Command: "vitest run --config production.ts"}}} + result := validationResult{Framework: "vitest", Preflight: &jestPreflight{Command: "npx vitest run", Verdict: verdict{Status: "compatible"}}, Compatibility: verdict{Status: "compatible"}, Features: []featureResult{{Name: "auto-retries", Status: "passed"}}} + setSingleConfigurationScope(root, &result, scope) + require.Equal(t, "passed", result.Configurations[0].Status) + require.Equal(t, "not exercised", result.Configurations[1].Status) + prepareValidationResult(root, &result) + require.False(t, result.Success) + require.False(t, result.LocalSuccess) + summary := summarizeValidation(result) + require.Equal(t, "INCOMPLETE", summary.Status) + require.Contains(t, strings.Join(summary.BlockingChecks, "\n"), "production.ts: not exercised") +} + +func TestAggregationKeepsFailedAndPendingConfigurations(t *testing.T) { + for _, status := range []string{"passed", "incomplete", "not exercised"} { + t.Run(status, func(t *testing.T) { + result := validationResult{Scope: &onboard.ValidationScope{}, Preflight: &jestPreflight{Verdict: verdict{Status: "compatible"}}, Compatibility: verdict{Status: "compatible"}, Features: []featureResult{{Name: "early-flake-detection", Status: "passed"}}, Configurations: []configurationResult{{TestCommand: onboard.TestCommand{Directory: ".", Command: "vitest run"}, Status: "passed"}, {TestCommand: onboard.TestCommand{Directory: ".", Command: "vitest run --coverage"}, Status: status}}} + prepareValidationResult(t.TempDir(), &result) + require.Equal(t, status == "passed", result.Success) + if status == "passed" { + result.Scope.Unresolved = []string{"unknown setup may run additional tests"} + prepareValidationResult(t.TempDir(), &result) + require.False(t, result.Success) + require.Contains(t, summarizeValidation(result).OnboardingResponse, "unresolved validation scope") + } + }) + } +} + +func TestConfigurationReportsRemainOnePairAndKeepCommandLabels(t *testing.T) { + root := t.TempDir() + models := []reportModel{{Tests: []reportTest{{Label: "suite › test", Name: "test"}}, Suites: []reportSuite{{Name: "suite"}}}, {Tests: []reportTest{{Label: "suite › test", Name: "test"}}, Suites: []reportSuite{{Name: "suite"}}}} + labelConfigurationModel(&models[0], "vitest run") + labelConfigurationModel(&models[1], "vitest run --config production.ts") + require.NoError(t, writeConfigurationHTML(root, models)) + code := 0 + result := validationResult{Framework: "vitest", Compatibility: verdict{Status: "compatible"}, Features: []featureResult{{Name: "early-flake-detection", Status: "passed"}}, Runs: []runSummary{{Name: "baseline", Configuration: "vitest run", Command: "vitest run", ExitCode: &code}, {Name: "reporting-only", Configuration: "vitest run", Command: "vitest run", Instrumented: true, ExitCode: &code}}} + require.NoError(t, finishValidation(&bytes.Buffer{}, root, result)) + html, err := os.ReadFile(htmlReportPath(root)) + require.NoError(t, err) + require.Contains(t, string(html), "vitest run --config production.ts") + require.NotContains(t, string(html), "intake/") + require.Contains(t, string(html), `href="testdrive.json"`) + entries, err := os.ReadDir(filepath.Dir(htmlReportPath(root))) + require.NoError(t, err) + require.Len(t, entries, 2) + require.NoError(t, finishValidation(&bytes.Buffer{}, root, validationResult{Framework: "vitest", CheckOnly: true, Preflight: &jestPreflight{Verdict: verdict{Status: "compatible"}}})) + data, err := os.ReadFile(validationPath(root)) + require.NoError(t, err) + var current validationResult + require.NoError(t, json.Unmarshal(data, ¤t)) + require.NotNil(t, current.Retained) + require.Equal(t, "vitest run", current.Retained.Result.Runs[0].Configuration) + require.False(t, current.Success) +} + +func TestCombinedHTMLPreservesCoverageLinksAndFullCommandOutput(t *testing.T) { + root := t.TempDir() + var models []reportModel + for i := range 2 { + facts := intake.Facts{ + TestEventCount: 1, TestCount: 1, CoverageLevel: "suite", + Tests: []intake.Test{{Module: "module", Suite: "shared", Name: "test", Status: "pass"}}, + SuiteCoverages: []intake.SuiteCoverage{{Module: "module", Suite: "shared", Files: []string{"source.js"}, CoveredTests: 1}}, + SlowSuites: []intake.SlowSuite{{Module: "module", Suite: "shared", Duration: 6 * time.Second}}, + } + model := buildReport(root, facts, false, reportRuntime{Framework: "Vitest", Tracer: "dd-trace", Command: fmt.Sprintf("vitest run --config variant-%d.ts", i), Output: strings.Repeat(fmt.Sprintf("variant-%d output\n", i), 200)}) + labelConfigurationModel(&model, model.Runtime.Command) + models = append(models, model) + } + for range 2 { + require.NoError(t, writeConfigurationHTML(root, models)) + page, err := os.ReadFile(htmlReportPath(root)) + require.NoError(t, err) + for i := range 2 { + require.Equal(t, 1, strings.Count(string(page), fmt.Sprintf(`id="test-detail-%d"`, i))) + require.Equal(t, 2, strings.Count(string(page), fmt.Sprintf(`data-open-test="test-detail-%d"`, i))) + require.Equal(t, 200, strings.Count(string(page), fmt.Sprintf("variant-%d output", i))) + require.Equal(t, 0, models[i].Suites[0].Tests[0].TestIndex, "aggregation must not mutate shared source rows") + } + require.Contains(t, string(page), "Suite coverage") + require.Contains(t, string(page), "source.js") + } +} + +func TestSourceRunDoesNotValidateCIPreparedConfiguration(t *testing.T) { + root := t.TempDir() + result := validationResult{Framework: "vitest", Preflight: &jestPreflight{Command: "vitest run"}, Compatibility: verdict{Status: "compatible"}, Features: []featureResult{{Name: "auto-retries", Status: "passed"}}} + scope := onboard.ValidationScope{Commands: []onboard.TestCommand{ + {Directory: ".", Command: "vitest run"}, + {Directory: ".", Command: "vitest run", UnvalidatedSetup: []string{"React dependency replacement"}}, + }} + setSingleConfigurationScope(root, &result, scope) + require.Equal(t, "passed", result.Configurations[0].Status) + require.Equal(t, "not exercised", result.Configurations[1].Status) + result.Success, result.LocalSuccess = true, true + applyScopeVerdict(&result) + require.False(t, result.Success) + require.False(t, result.LocalSuccess) +} + +func TestAllDoesNotExecuteAConfigurationWithoutItsCISetup(t *testing.T) { + drive := preparedTestdrive(t) + root := drive.repositoryRoot + require.NoError(t, os.MkdirAll(filepath.Join(root, ".github/workflows"), 0755)) + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"jest"}}`) + requireWriteFile(t, filepath.Join(root, ".github/workflows/test.yml"), "jobs:\n test:\n steps:\n - run: sed -i~ 's/src/dist/e' jest.config.js\n - run: npm test\n") + executor := &buildExecutor{t: t} + drive.executor = executor + require.ErrorContains(t, drive.runAllConfigurations(t.Context(), &bytes.Buffer{}), "requires separate CI setup") + require.Empty(t, executor.commands, "must not certify source tests as a built-package configuration") + data, err := os.ReadFile(validationPath(root)) + require.NoError(t, err) + var result validationResult + require.NoError(t, json.Unmarshal(data, &result)) + require.Equal(t, "INCOMPLETE", result.Summary.Status) + require.Equal(t, "not exercised", result.Configurations[0].Status) + require.Empty(t, result.Runs) + require.NotEmpty(t, result.Configurations[0].UnvalidatedSetup) +} diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index a78d4e06..fa2b5530 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -32,9 +32,9 @@ func TestPublicFrameworkTestdrives(t *testing.T) { name, manifest, command string files map[string]string }{ - {"jest", `{"scripts":{"test":"jest"},"devDependencies":{"jest":"30.5.1"}}`, "npm test", map[string]string{"one.test.js": `test('adds', () => expect(1+1).toBe(2));`}}, + {"jest", `{"scripts":{"test":"jest"},"devDependencies":{"jest":"30.5.1"}}`, "npm test -- --runInBand -- one.test.js", map[string]string{"one.test.js": `test('adds', () => expect(1+1).toBe(2));`}}, {"mocha", `{"scripts":{"test":"mocha"},"devDependencies":{"mocha":"11.7.5"}}`, "npm test", map[string]string{"test/one.js": `const assert = require('node:assert'); it('adds', () => assert.equal(1+1,2));`}}, - {"vitest", `{"type":"module","scripts":{"test":"vitest run"},"devDependencies":{"vitest":"3.2.4"}}`, "npm test", map[string]string{"one.test.js": `import {test,expect} from 'vitest'; test('adds', () => expect(1+1).toBe(2));`}}, + {"vitest", `{"type":"module","scripts":{"test":"vitest run"},"devDependencies":{"vitest":"4.1.6"}}`, "npm test", map[string]string{"one.test.js": `import {test,expect} from 'vitest'; test('adds', () => expect(1+1).toBe(2));`}}, {"playwright", `{"scripts":{"test":"playwright test"},"devDependencies":{"@playwright/test":"1.55.1"}}`, "npm test", map[string]string{"one.spec.js": `const {test,expect} = require('@playwright/test'); test('adds', () => expect(1+1).toBe(2));`}}, {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, {"cypress", `{"scripts":{"test":"cypress run"},"devDependencies":{"cypress":"15.1.0"}}`, "npm test", map[string]string{"cypress.config.js": `module.exports={e2e:{supportFile:false,setupNodeEvents(on,config){on('task',{answer:()=>42});on('after:run',()=>{require('node:fs').writeFileSync('original-hook.txt','ran');});return config;}}};`, "cypress/e2e/one.cy.js": `it('preserves hooks', () => { cy.task('answer').should('equal',42); });`}}, @@ -80,18 +80,58 @@ func TestPublicFrameworkTestdrives(t *testing.T) { } onboard := integrationCommand(t, ctx, root, env, binary, "onboard") require.Contains(t, onboard, "datadog/test-visibility-github-action@v3") - output := integrationCommand(t, ctx, root, env, binary, "testdrive", "--yes") - require.Contains(t, output, "Test events received.") - require.Contains(t, output, "Open report:") - reports, err := filepath.Glob(filepath.Join(root, ".testoptimization", "testdrive", "*", "report.html")) + args := []string{"testdrive", "--yes"} + if fixture.name == "jest" { + // The validation contract also requires instrumented CI configuration. + // Resolve once so local and CI metadata refer to the same release. + version := strings.TrimSpace(integrationCommand(t, ctx, root, env, "npm", "view", "dd-trace", "version")) + node := strings.TrimSpace(integrationCommand(t, ctx, root, env, "node", "--version")) + integrationFile(t, root, ".github/workflows/test.yml", "name: tests\non: push\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/setup-node@v4\n with:\n node-version: '"+strings.TrimPrefix(node, "v")+"'\n - uses: datadog/test-visibility-github-action@v3\n with:\n languages: js\n js-tracer-version: '"+version+"'\n - run: "+fixture.command+"\n env:\n NODE_OPTIONS: \"-r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}\"\n") + args = append(args, "--tracer-version", version) + } + command := exec.CommandContext(ctx, binary, args...) + command.Dir = root + command.Env = append(os.Environ(), env...) + rawOutput, runErr := command.CombinedOutput() + output := string(rawOutput) + switch fixture.name { + case "jest": + require.NoError(t, runErr, output) + t.Log(strings.TrimSpace(output)) + // Reuse the exact release from the fallback run, and exercise an + // explicit wrapper separator with the installed-tracer preview. + integrationCommand(t, ctx, root, env, "npm", "install", "--no-save", "--package-lock=false", "--no-audit", "--no-fund", "dd-trace@"+args[len(args)-1]) + output = integrationCommand(t, ctx, root, env, binary, "testdrive", "--yes", "--command", "npx -- jest --runInBand -- one.test.js") + require.Contains(t, output, "reuse installed dd-trace@") + require.Contains(t, output, "no installation is needed") + require.NotContains(t, output, "npm install") + probes, err := filepath.Glob(filepath.Join(root, "ddtest*")) + require.NoError(t, err) + require.Empty(t, probes) + case "vitest": + require.Error(t, runErr, output) // Fixture deliberately has no CI installation. + require.Contains(t, output, "No observed behavioral regression") + require.Contains(t, output, "Feature auto-retries: passed") + require.Contains(t, output, "CI runtime incompatible") + default: + require.Error(t, runErr, output) + require.Contains(t, output, "unvalidated") + } + require.Contains(t, output, "Results JSON:") + reports, err := filepath.Glob(filepath.Join(root, ".testoptimization", "testdrive.json")) require.NoError(t, err) require.Len(t, reports, 1) - contents, err := os.ReadFile(reports[0]) + artifacts, err := os.ReadDir(filepath.Join(root, ".testoptimization")) require.NoError(t, err) - require.Contains(t, string(contents), "Test events received.") - traffic, err := filepath.Glob(filepath.Join(filepath.Dir(reports[0]), "intake", "*citestcycle.json")) + require.Len(t, artifacts, 2, "only HTML and compact JSON reports should remain") + html, err := filepath.Glob(filepath.Join(root, ".testoptimization", "*.html")) + require.NoError(t, err) + require.Equal(t, []string{filepath.Join(root, ".testoptimization", "report.html")}, html) + contents, err := os.ReadFile(html[0]) require.NoError(t, err) - require.NotEmpty(t, traffic) + require.Contains(t, string(contents), "Test events received.") + require.Contains(t, string(contents), "

Test report

") + require.Contains(t, output, "Open report:") for name, contents := range before { if (fixture.name == "rspec" || fixture.name == "minitest") && (name == "Gemfile" || name == "Gemfile.lock") { continue // bundle add updates Ruby dependency files. diff --git a/internal/testdrive/intake/events.go b/internal/testdrive/intake/events.go index ef44e286..6de45da9 100644 --- a/internal/testdrive/intake/events.go +++ b/internal/testdrive/intake/events.go @@ -235,3 +235,41 @@ func truthy(value any) bool { return integer(value) != 0 } } + +// Event retains the identity and feature tags needed to verify a scenario, +// including suite events when the tracer skips an entire Jest file. +type Event struct { + Type string `json:"type"` + Tags map[string]string `json:"tags"` +} + +func (s *Server) events() ([]Event, error) { + var events []Event + for _, request := range s.Requests() { + if request.Method != http.MethodPost || request.Path != constants.TestCycleURLPath { + continue + } + body, err := uncompressRequestBody(request) + if err != nil { + return nil, err + } + payload, _, err := msgp.ReadMapStrIntfBytes(body, nil) + if err != nil { + return nil, err + } + values, _ := payload["events"].([]any) + for _, value := range values { + event, _ := value.(map[string]any) + content, _ := event["content"].(map[string]any) + metadata, _ := content["meta"].(map[string]any) + tags := make(map[string]string) + for key, value := range metadata { + if v, ok := value.(string); ok { + tags[key] = v + } + } + events = append(events, Event{Type: text(event["type"]), Tags: tags}) + } + } + return events, nil +} diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go index cebec56e..741c2f3e 100644 --- a/internal/testdrive/intake/findings.go +++ b/internal/testdrive/intake/findings.go @@ -79,6 +79,9 @@ type SlowSuite struct { // Facts contains the facts shown in the testdrive report. type Facts struct { + SettingsRequests int + SkippableRequests int + Events []Event ConfigurationErrors []string EmptyCoverageEntryCount int MissingCoverage bool @@ -110,6 +113,19 @@ func (s *Server) Facts() (Facts, error) { } findings := Facts{TestEventCount: len(tests), CoverageLevel: coverageLevel(coverages), EmptyCoverageEntryCount: emptyEntries} + s.requestsMu.Lock() + for _, request := range s.requests { + if request.Method != http.MethodPost { + continue + } + switch request.Path { + case constants.SettingsURLPath: + findings.SettingsRequests++ + case constants.SkippableTestsURLPath: + findings.SkippableRequests++ + } + } + s.requestsMu.Unlock() findings.Tests, findings.FailedTests, findings.FlakyTests, findings.SlowTests, findings.TestDurationMedian = analyzeTests(tests, coverages, findings.CoverageLevel) if findings.CoverageLevel == "suite" { findings.SuiteCoverages = suiteCoverages(tests, coverages) @@ -120,6 +136,10 @@ func (s *Server) Facts() (Facts, error) { findings.MissingCoverage = len(tests) > 0 && len(coverages) == 0 && emptyEntries == 0 findings.BroadCoverage, findings.CoveredFilesMedian = analyzeCoverage(tests, coverages, findings.CoverageLevel) findings.ConfigurationErrors, err = s.configurationErrors() + if err != nil { + return findings, err + } + findings.Events, err = s.events() return findings, err } diff --git a/internal/testdrive/intake/server.go b/internal/testdrive/intake/server.go index 6acf0ebb..d1e68df6 100644 --- a/internal/testdrive/intake/server.go +++ b/internal/testdrive/intake/server.go @@ -81,6 +81,11 @@ type Server struct { // Start starts an HTTP intake on a kernel-assigned loopback port and stores // every request under the testdrive session directory. func Start(sessionDirectory string) (*Server, error) { + return StartScenario(sessionDirectory, Scenario{}) +} + +// StartScenario starts an isolated intake with fixed responses for one experiment. +func StartScenario(sessionDirectory string, scenario Scenario) (*Server, error) { intakeDirectory := filepath.Join(sessionDirectory, intakeDirectoryName) if err := os.MkdirAll(intakeDirectory, 0755); err != nil { return nil, fmt.Errorf("create local testdrive intake directory: %w", err) @@ -97,7 +102,7 @@ func Start(sessionDirectory string) (*Server, error) { directory: intakeDirectory, } httpServer := &http.Server{ - Handler: server.recordRequests(newHandler()), + Handler: server.recordRequests(scenarioHandler(scenario)), ReadHeaderTimeout: 10 * time.Second, } server.server = httpServer diff --git a/internal/testdrive/intake/settings.go b/internal/testdrive/intake/settings.go index dc5a0a36..8ad2aa66 100644 --- a/internal/testdrive/intake/settings.go +++ b/internal/testdrive/intake/settings.go @@ -20,12 +20,25 @@ const ( testdriveCorrelationID = "ddtest-testdrive" ) -func newHandler() http.Handler { +// Scenario configures exactly one local feature experiment. The zero value is reporting-only. +type Scenario struct { + Feature string + Module string + Suite string + Test string + // SourceFile is the repository-relative path used by Jest suite skipping. + // Suite remains the reported identity used by test management. + SourceFile string +} + +func newHandler() http.Handler { return scenarioHandler(Scenario{}) } + +func scenarioHandler(scenario Scenario) http.Handler { mux := http.NewServeMux() - mux.HandleFunc("POST "+constants.SettingsURLPath, handleSettings) + mux.HandleFunc("POST "+constants.SettingsURLPath, func(w http.ResponseWriter, r *http.Request) { handleScenarioSettings(w, r, scenario) }) mux.HandleFunc("POST "+constants.KnownTestsURLPath, handleKnownTests) - mux.HandleFunc("POST "+constants.SkippableTestsURLPath, handleSkippableTests) - mux.HandleFunc("POST "+constants.TestManagementTestsURLPath, handleTestManagement) + mux.HandleFunc("POST "+constants.SkippableTestsURLPath, func(w http.ResponseWriter, r *http.Request) { handleSkippableTests(w, r, scenario) }) + mux.HandleFunc("POST "+constants.TestManagementTestsURLPath, func(w http.ResponseWriter, r *http.Request) { handleTestManagement(w, r, scenario) }) mux.HandleFunc("POST "+constants.SearchCommitsURLPath, func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, map[string]any{"data": []any{}}) }) @@ -49,7 +62,7 @@ func newHandler() http.Handler { return mux } -func handleSettings(w http.ResponseWriter, request *http.Request) { +func handleScenarioSettings(w http.ResponseWriter, request *http.Request, scenario Scenario) { var settingsRequest api.SettingsRequest if err := json.NewDecoder(request.Body).Decode(&settingsRequest); err != nil { http.Error(w, "invalid settings request", http.StatusBadRequest) @@ -65,25 +78,19 @@ func handleSettings(w http.ResponseWriter, request *http.Request) { response.Data.ID = responseID response.Data.Type = constants.SettingsResponseType response.Data.Attributes = api.SettingsResponseData{ - CodeCoverage: true, - CoverageReportUploadEnabled: true, - // Testdrive runs the whole suite and disables git upload. Some tracers - // wait for that upload before fetching skippable tests, so advertising - // skipping here can prevent Jest from ever starting. - TestsSkipping: false, + CodeCoverage: true, ItrEnabled: true, - ImpactedTestsEnabled: true, - FlakyTestRetriesEnabled: true, - DIEnabled: true, - KnownTestsEnabled: true, + TestsSkipping: scenario.Feature == "skipping", + FlakyTestRetriesEnabled: scenario.Feature == "auto-retries", + KnownTestsEnabled: scenario.Feature == "early-flake-detection", EarlyFlakeDetection: api.EarlyFlakeDetectionSettings{ - Enabled: true, - SlowTestRetries: api.SlowTestRetries{FiveS: 1, TenS: 1, ThirtyS: 1, FiveM: 1}, + Enabled: scenario.Feature == "early-flake-detection", + SlowTestRetries: api.SlowTestRetries{FiveS: 2, TenS: 2, ThirtyS: 2, FiveM: 2}, FaultySessionThreshold: 100, }, TestManagement: api.TestManagementSettings{ - Enabled: true, - AttemptToFixRetries: 1, + Enabled: scenario.Feature == "quarantine" || scenario.Feature == "disabled" || scenario.Feature == "attempt-to-fix", + AttemptToFixRetries: 2, }, } @@ -108,9 +115,13 @@ func handleKnownTests(w http.ResponseWriter, _ *http.Request) { }) } -func handleSkippableTests(w http.ResponseWriter, _ *http.Request) { +func handleSkippableTests(w http.ResponseWriter, _ *http.Request, scenario Scenario) { + data := []any{} + if scenario.Feature == "skipping" && scenario.SourceFile != "" { + data = append(data, map[string]any{"type": "suite", "attributes": map[string]any{"suite": scenario.SourceFile}}) + } writeJSON(w, map[string]any{ - "data": []any{}, + "data": data, "meta": map[string]any{ "correlation_id": testdriveCorrelationID, "coverage": map[string]any{}, @@ -118,12 +129,17 @@ func handleSkippableTests(w http.ResponseWriter, _ *http.Request) { }) } -func handleTestManagement(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, map[string]any{ - "data": map[string]any{ - "attributes": map[string]any{"modules": map[string]any{}}, - }, - }) +func handleTestManagement(w http.ResponseWriter, _ *http.Request, scenario Scenario) { + modules := map[string]any{} + property := map[string]string{"quarantine": "quarantined", "disabled": "disabled", "attempt-to-fix": "attempt_to_fix"}[scenario.Feature] + if property != "" { + modules[scenario.Module] = map[string]any{"suites": map[string]any{ + scenario.Suite: map[string]any{"tests": map[string]any{ + scenario.Test: map[string]any{"properties": map[string]any{property: true}}, + }}, + }} + } + writeJSON(w, map[string]any{"data": map[string]any{"attributes": map[string]any{"modules": modules}}}) } func writeJSON(w http.ResponseWriter, response any) { diff --git a/internal/testdrive/intake/settings_test.go b/internal/testdrive/intake/settings_test.go index 56a99d0f..77c3709c 100644 --- a/internal/testdrive/intake/settings_test.go +++ b/internal/testdrive/intake/settings_test.go @@ -46,14 +46,14 @@ func TestSettingsEnablesTestOptimizationCoverage(t *testing.T) { require.True(t, settings.Data.Attributes.CodeCoverage) require.False(t, settings.Data.Attributes.TestsSkipping) require.False(t, settings.Data.Attributes.RequireGit) - require.True(t, settings.Data.Attributes.CoverageReportUploadEnabled) - require.True(t, settings.Data.Attributes.ImpactedTestsEnabled) - require.True(t, settings.Data.Attributes.FlakyTestRetriesEnabled) - require.True(t, settings.Data.Attributes.DIEnabled) - require.True(t, settings.Data.Attributes.KnownTestsEnabled) - require.True(t, settings.Data.Attributes.EarlyFlakeDetection.Enabled) - require.Equal(t, 1, settings.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.FiveS) - require.True(t, settings.Data.Attributes.TestManagement.Enabled) + require.False(t, settings.Data.Attributes.CoverageReportUploadEnabled) + require.False(t, settings.Data.Attributes.ImpactedTestsEnabled) + require.False(t, settings.Data.Attributes.FlakyTestRetriesEnabled) + require.False(t, settings.Data.Attributes.DIEnabled) + require.False(t, settings.Data.Attributes.KnownTestsEnabled) + require.False(t, settings.Data.Attributes.EarlyFlakeDetection.Enabled) + require.Equal(t, 2, settings.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.FiveS) + require.False(t, settings.Data.Attributes.TestManagement.Enabled) } func TestAdvancedFeatureEndpointsReturnSafeEmptyDatasets(t *testing.T) { @@ -102,3 +102,49 @@ func TestGitNegotiation(t *testing.T) { require.Equal(t, http.StatusNotFound, unknown.Code) require.Contains(t, unknown.Body.String(), "unsupported") } + +func TestScenarioResponsesEnableOnlySelectedBehavior(t *testing.T) { + for _, feature := range []string{"", "auto-retries", "early-flake-detection", "skipping", "quarantine", "disabled", "attempt-to-fix"} { + t.Run(feature, func(t *testing.T) { + handler := scenarioHandler(Scenario{Feature: feature, Module: "custom-module", Suite: "../../src/probe.test.js", SourceFile: "src/probe.test.js", Test: "probe"}) + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, constants.SettingsURLPath, bytes.NewBufferString(`{"data":{}}`))) + require.Equal(t, http.StatusOK, response.Code) + var settings api.SettingsResponse + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &settings)) + attributes := settings.Data.Attributes + require.Equal(t, feature == "auto-retries", attributes.FlakyTestRetriesEnabled) + require.Equal(t, feature == "early-flake-detection", attributes.EarlyFlakeDetection.Enabled) + require.Equal(t, feature == "skipping", attributes.TestsSkipping) + require.Equal(t, feature == "quarantine" || feature == "disabled" || feature == "attempt-to-fix", attributes.TestManagement.Enabled) + require.False(t, attributes.ImpactedTestsEnabled) + require.False(t, attributes.DIEnabled) + response = httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, constants.SkippableTestsURLPath, bytes.NewBufferString(`{}`))) + if feature == "skipping" { + require.Contains(t, response.Body.String(), `"suite":"src/probe.test.js"`) + require.NotContains(t, response.Body.String(), "../../") + } else { + require.Contains(t, response.Body.String(), `"data":[]`) + } + response = httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, constants.TestManagementTestsURLPath, bytes.NewBufferString(`{}`))) + if attributes.TestManagement.Enabled { + require.Contains(t, response.Body.String(), `"custom-module"`) + require.Contains(t, response.Body.String(), `"../../src/probe.test.js"`) + require.Contains(t, response.Body.String(), `"probe"`) + } else { + require.Contains(t, response.Body.String(), `"modules":{}`) + } + }) + } +} + +func TestSkippingWithoutSourceDoesNotReturnTheSuiteIdentity(t *testing.T) { + response := httptest.NewRecorder() + handler := scenarioHandler(Scenario{Feature: "skipping", Suite: "../../src/probe.test.js"}) + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, constants.SkippableTestsURLPath, bytes.NewBufferString(`{}`))) + require.Equal(t, http.StatusOK, response.Code) + require.Contains(t, response.Body.String(), `"data":[]`) + require.NotContains(t, response.Body.String(), "probe.test.js") +} diff --git a/internal/testdrive/javascript.go b/internal/testdrive/javascript.go index db3939bb..36f4318b 100644 --- a/internal/testdrive/javascript.go +++ b/internal/testdrive/javascript.go @@ -2,6 +2,7 @@ package testdrive import ( "encoding/json" + "github.com/kballard/go-shellquote" "os" "os/exec" "path/filepath" @@ -21,7 +22,10 @@ func javascriptEnvironment(ciInitPath string) map[string]string { } func stripDatadogNodeOptions(value string) string { - fields := strings.Fields(value) + fields, err := shellquote.Split(value) + if err != nil { + return value + } kept := make([]string, 0, len(fields)) for index := 0; index < len(fields); index++ { field := fields[index] @@ -40,6 +44,9 @@ func stripDatadogNodeOptions(value string) string { if strings.HasPrefix(field, "-r") && isDatadogNodePreload(strings.TrimPrefix(field, "-r")) { continue } + if strings.ContainsAny(field, " \t\r\n\"") { + field = strconv.Quote(field) + } kept = append(kept, field) } return strings.Join(kept, " ") @@ -47,7 +54,7 @@ func stripDatadogNodeOptions(value string) string { func isDatadogNodePreload(value string) bool { value = strings.Trim(value, `"'`) - return value == "dd-trace/ci/init" || strings.HasSuffix(filepath.ToSlash(value), "/dd-trace/ci/init.js") || + return value == "dd-trace/ci/init" || value == "dd-trace/register.js" || strings.HasSuffix(filepath.ToSlash(value), "/dd-trace/ci/init.js") || strings.HasSuffix(filepath.ToSlash(value), "/dd-trace/register.js") } diff --git a/internal/testdrive/jest.go b/internal/testdrive/jest.go new file mode 100644 index 00000000..e5727d72 --- /dev/null +++ b/internal/testdrive/jest.go @@ -0,0 +1,413 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/kballard/go-shellquote" +) + +type jestTest struct { + File string `json:"file"` + Name string `json:"name"` + Status string `json:"status"` + Failure string `json:"failure,omitempty"` + + telemetryName *string +} + +type jestResults struct { + TestResults []struct { + Name string `json:"name"` + Message string `json:"message"` + Status string `json:"status"` + AssertionResults []struct { + FullName string `json:"fullName"` + Title *string `json:"title"` + AncestorTitles []string `json:"ancestorTitles"` + Status string `json:"status"` + FailureMessages []string `json:"failureMessages"` + } `json:"assertionResults"` + } `json:"testResults"` +} + +var ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) + +func failureSignature(message string) string { + var lines []string + for _, line := range strings.Split(ansiEscape.ReplaceAllString(message, ""), "\n") { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "at ") { + lines = append(lines, line) + } + } + return strings.Join(lines, "\n") +} + +func readJestResults(path string, run *validationRun) { readRunnerResults(path, "Jest", run) } + +func readRunnerResults(path, label string, run *validationRun) { + data, err := os.ReadFile(path) + if err != nil { + run.ResultError = label + " did not produce its JSON results" + return + } + var result jestResults + if err := json.Unmarshal(data, &result); err != nil { + run.ResultError = label + " JSON results could not be decoded" + return + } + if result.TestResults == nil { + run.ResultError = label + " JSON results are missing the testResults array" + return + } + for _, suite := range result.TestResults { + if len(suite.AssertionResults) == 0 && suite.Status == "failed" { + run.SuiteErrors = append(run.SuiteErrors, suite.Name+": "+failureSignature(suite.Message)) + } + for _, test := range suite.AssertionResults { + parsed := jestTest{File: filepath.Clean(suite.Name), Name: test.FullName, Status: test.Status, + Failure: failureSignature(strings.Join(test.FailureMessages, "\n"))} + if label == "Vitest" && test.Title != nil && test.AncestorTitles != nil { + // Vitest's JSON fullName omits an empty test title; dd-trace + // includes it and skips empty ancestor titles. Preserve the + // runner's identity for outcome comparison and derive only the + // expected telemetry name, without trimming meaningful spaces. + var names []string + for _, ancestor := range test.AncestorTitles { + if ancestor != "" { + names = append(names, ancestor) + } + } + name := strings.Join(append(names, *test.Title), " ") + parsed.telemetryName = &name + } + run.Tests = append(run.Tests, parsed) + } + } + slices.Sort(run.SuiteErrors) +} + +func appendJestArgs(command string, args []string, additions ...string) []string { + args = slices.Clone(args) + if filepath.Base(command) == "npm" && !slices.Contains(args, "--") { + args = append(args, "--") + } + if index := jestSeparator(command, args); index >= 0 { + return slices.Insert(args, index, additions...) + } + return append(args, additions...) +} + +func jestSeparator(command string, args []string) int { + index := framework.FrameworkSeparator(command, args, "jest") + // npm consumes its first separator when forwarding script arguments. + if filepath.Base(command) == "npm" && index >= 0 && index == slices.Index(args, "--") { + if next := slices.Index(args[index+1:], "--"); next >= 0 { + return index + 1 + next + } + return -1 + } + return index +} + +func withJestProbe(command string, args []string, probe string) []string { + if index := jestSeparator(command, args); index >= 0 { + args = args[:index+1] + } + return appendJestArgs(command, args, "--runTestsByPath", probe) +} + +func probeJestArgs(command string, args []string) []string { + // Jest ignores duplicate JSON options instead of taking the last one. + var result []string + for i := 0; i < len(args); i++ { + if args[i] == "--coverageThreshold" || args[i] == "--coverage-threshold" { + i++ + continue + } + if strings.HasPrefix(args[i], "--coverageThreshold=") || strings.HasPrefix(args[i], "--coverage-threshold=") { + continue + } + result = append(result, args[i]) + } + return appendJestArgs(command, result, "--coverageThreshold={}") +} + +func (t *Testdrive) runJest(ctx context.Context, output io.Writer, session *Session, preload, name string, instrumented bool, scenario intake.Scenario, probePath, probeMode string) (run validationRun, runErr error) { + run.Name = name + run.Instrumented = instrumented + run.ProbeMode = probeMode + run.root = t.repositoryRoot + if err := ctx.Err(); err != nil { + return run, err + } + directory := filepath.Join(session.Directory(), name) + if err := os.MkdirAll(directory, 0700); err != nil { + return run, err + } + // Even the baseline receives a local endpoint and disabled uploads, so a + // tracer imported by the project's own setup cannot contact Datadog. + server, err := t.startIntake(directory, scenario) + if err != nil { + return run, err + } + closed := false + defer func() { + if !closed { + runErr = errors.Join(runErr, server.Close()) + } + }() + env := t.environment(preload, server.URL(), session.ID()+" "+name) + configureScenarioEnvironment(env, scenario.Feature) + if !instrumented { + env["NODE_OPTIONS"] = stripDatadogNodeOptions(os.Getenv("NODE_OPTIONS")) + env["DD_CIVISIBILITY_ENABLED"] = "false" + env["DD_TRACE_ENABLED"] = "false" + } + // Package scripts can already set coverageDirectory. Jest treats duplicate + // string options as arrays, so normalize only our override before parsing. + coverageDirectory := filepath.Join(directory, "coverage") + if err := redirectJestCoverage(directory, coverageDirectory, env); err != nil { + return run, err + } + resultPath := filepath.Join(directory, "jest-results.json") + args := appendJestArgs(t.command, t.args, "--json", "--outputFile", resultPath, "--coverageDirectory", coverageDirectory) + if probePath != "" { + // A single synthetic test cannot meet whole-project coverage thresholds. + // Keep coverage collection enabled, including for Test Impact Analysis. + args = withJestProbe(t.command, args, probePath) + args = appendJestArgs(t.command, probeJestArgs(t.command, args), "--testNamePattern", "^"+probeName+"$") + run.ProbeCoverageThresholdsDisabled = true + env["DDTEST_PROBE_MODE"] = probeMode + env["DDTEST_JEST_PROBE"] = "1" + run.ProbeLintSkipped = strings.Contains(strings.Join(t.args, " "), "DDTEST_JEST_PROBE") + } + _, _ = fmt.Fprintf(output, "Running %s...\n", name) + run.Command = shellquote.Join(append([]string{t.command}, args...)...) + commandOutput, commandErr := t.executor.CombinedOutput(ctx, t.command, args, env) + run.commandOutput, run.commandError = string(commandOutput), commandErr + run.ExitCode = commandExitCode(commandErr) + if commandErr != nil && probePath == "" { + writeCommandFailure(output, displayName(t.framework.Name())+" "+name, commandOutput) + } + if err := os.WriteFile(filepath.Join(directory, testOutputFilename), commandOutput, 0600); err != nil { + return run, fmt.Errorf("save test output: %w", err) + } + closeErr := server.Close() + closed = true + if closeErr != nil { + return run, closeErr + } + if err := ctx.Err(); err != nil { + return run, err + } + run.Facts, err = server.Facts() + if err != nil { + return run, err + } + readJestResults(resultPath, &run) + if scenario.Feature == "skipping" { + run.Skipping = diagnoseSkipping(run, scenario) + } + if run.ExitCode != 0 || run.ResultError != "" || len(run.SuiteErrors) > 0 { + run.Diagnostic = commandDiagnostic(commandOutput) + } + return run, nil +} + +// Preserve only the bounded tail where Jest normally prints its final errors, +// including coverage failures which do not appear in its JSON test results. +func commandDiagnostic(output []byte) string { + text := []rune(failureSignature(string(output))) + const limit = 1024 + if len(text) > limit { + const prefix = "[truncated] …" + return prefix + string(text[len(text)-(limit-len([]rune(prefix))):]) + } + return string(text) +} + +func configureScenarioEnvironment(env map[string]string, feature string) { + // Coverage requires ITR negotiation; tests_skipping is separately disabled by + // the reporting-only settings response. No skippable tests are returned there. + env["DD_CIVISIBILITY_ITR_ENABLED"] = "true" + env["DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED"] = fmt.Sprint(feature == "early-flake-detection") + env["DD_TEST_EARLY_FLAKE_DETECTION_RETRY_COUNT"] = "2" + env["DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"] = fmt.Sprint(feature == "auto-retries") + env["DD_CIVISIBILITY_FLAKY_RETRY_COUNT"] = "2" + env["DD_TEST_MANAGEMENT_ENABLED"] = fmt.Sprint(feature == "quarantine" || feature == "disabled" || feature == "attempt-to-fix") + env["DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES"] = "2" +} + +func executedTests(run validationRun) int { + count := 0 + for _, test := range run.Tests { + if test.Status == "passed" || test.Status == "failed" { + count++ + } + } + return count +} + +func outcomeKeys(run validationRun) []string { + keys := []string{fmt.Sprintf("exit=%d", run.ExitCode), "result=" + run.ResultError} + keys = append(keys, run.SuiteErrors...) + for _, test := range run.Tests { + keys = append(keys, test.File+" › "+test.Name+": "+test.Status+"\n"+test.Failure) + } + slices.Sort(keys) + return keys +} + +func compareJest(baseline, instrumented validationRun) verdict { + if baseline.ResultError != "" || executedTests(baseline) == 0 { + return verdict{Status: "inconclusive", Reason: "The uninstrumented run did not produce results for any executed tests; check project setup using the recorded baseline command."} + } + if baseline.Facts.TestEventCount > 0 { + return verdict{Status: "inconclusive", Reason: "The baseline emitted test telemetry despite instrumentation being disabled; a clean comparison was not established."} + } + left, right := outcomeKeys(baseline), outcomeKeys(instrumented) + if !slices.Equal(left, right) { + differences := []string{} + counts := make(map[string]int) + for _, key := range left { + counts[key]++ + } + for _, key := range right { + counts[key]-- + } + for key, count := range counts { + if count == 0 { + continue + } + label, _, _ := strings.Cut(key, "\n") + side := "Without instrumentation only" + if count < 0 { + side = "With instrumentation only" + } + differences = append(differences, side+": "+label) + } + slices.Sort(differences) + return verdict{Status: "suspected regression", Reason: "Uninstrumented and reporting-only results differ.", Differences: differences} + } + if telemetry := verifyJestTelemetry(instrumented); telemetry.Status != "" { + return telemetry + } + return verdict{Status: "compatible", Reason: "No observed behavioral regression: test identities, outcomes, failure details, and exit code match. Existing test failures are acceptable; reporting telemetry was verified."} +} + +func verifyJestTelemetry(run validationRun) verdict { + if run.Facts.TestEventCount == 0 { + return verdict{Status: "inconclusive", Reason: "The instrumented run sent no test events; matching test outcomes do not prove instrumentation worked."} + } + if len(run.Facts.ConfigurationErrors) > 0 { + return verdict{Status: "inconclusive", Reason: "The tracer reported configuration errors: " + strings.Join(run.Facts.ConfigurationErrors, ", ")} + } + type identity struct{ file, name, status string } + expected := make(map[identity]int) + for _, test := range run.Tests { + status := map[string]string{"passed": "pass", "failed": "fail"}[test.Status] + if status != "" { + name := test.Name + if test.telemetryName != nil { + name = *test.telemetryName + } + expected[identity{test.File, name, status}]++ + } + } + observed := make(map[identity]int) + for _, test := range run.Facts.Tests { + file := test.SourceFile + if file == "" { + file = test.Suite + } + if !filepath.IsAbs(file) { + file = filepath.Join(run.root, file) + } + file = filepath.Clean(file) + for _, attempt := range test.Attempts { + if attempt.Status == "pass" || attempt.Status == "fail" { + observed[identity{file, test.Name, attempt.Status}]++ + } + if attempt.Retry { + return verdict{Status: "inconclusive", Reason: "Unexpected retry telemetry in the reporting-only run."} + } + } + } + var differences []string + for key, count := range expected { + if count > observed[key] { + differences = append(differences, fmt.Sprintf("Missing telemetry: file %q, test %q, status %q (count %d)", key.file, key.name, key.status, count-observed[key])) + } + } + for key, count := range observed { + if count > expected[key] { + differences = append(differences, fmt.Sprintf("Unexpected telemetry: file %q, test %q, status %q (count %d)", key.file, key.name, key.status, count-expected[key])) + } + } + if len(differences) > 0 { + slices.Sort(differences) + return verdict{Status: "inconclusive", Reason: "Reported telemetry does not match the executed tests.", Differences: differences} + } + return verdict{} +} + +func (t *Testdrive) runJavaScriptValidation(ctx context.Context, output io.Writer, session *Session, preload string, result *validationResult) error { + run := t.runJest + if t.framework.Name() == "vitest" { + run = t.runVitest + } + baseline, err := run(ctx, output, session, preload, "baseline", false, intake.Scenario{}, "", "") + result.Runs = append(result.Runs, baseline.summary()) + if err != nil { + return err + } + instrumented, err := run(ctx, output, session, preload, "reporting-only", true, intake.Scenario{}, "", "") + result.Runs = append(result.Runs, instrumented.summary()) + if err != nil { + return err + } + if err := t.writeHTMLReport(output, result, instrumented); err != nil { + return err + } + result.Compatibility = compareJest(baseline, instrumented) + if result.Compatibility.Status == "suspected regression" { + baselineAgain, err := run(ctx, output, session, preload, "baseline-repeat", false, intake.Scenario{}, "", "") + result.Runs = append(result.Runs, baselineAgain.summary()) + if err != nil { + return err + } + instrumentedAgain, err := run(ctx, output, session, preload, "reporting-only-repeat", true, intake.Scenario{}, "", "") + result.Runs = append(result.Runs, instrumentedAgain.summary()) + if err != nil { + return err + } + if err := t.writeHTMLReport(output, result, instrumentedAgain); err != nil { + return err + } + result.Compatibility = compareRepeatedJavaScript(baseline, instrumented, baselineAgain, instrumentedAgain) + } + if t.framework.Name() == "vitest" { + return t.runVitestFeatures(ctx, output, session, preload, baseline, result) + } + if err := t.runJestFeatures(ctx, output, session, preload, result); err != nil { + return err + } + return nil +} diff --git a/internal/testdrive/jest_coverage.go b/internal/testdrive/jest_coverage.go new file mode 100644 index 00000000..79756c65 --- /dev/null +++ b/internal/testdrive/jest_coverage.go @@ -0,0 +1,49 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" +) + +// Applied equally to baseline and instrumented processes. npm can introduce +// duplicate flags after expanding nested package scripts; Go cannot remove those +// from the outer command. Keep the recorded temporary output override exactly +// once, without changing the package script or any other Jest option. +const coverageArguments = ` +const target = %s; +const args = process.argv; +const matches = []; +for (let i = 2; i < args.length; i++) { + const match = /^(--coverageDirectory|--coverage-directory)(?:=(.*))?$/.exec(args[i]); + if (!match) continue; + const index = i; + const value = match[2] === undefined ? args[++i] : match[2]; + matches.push({index, count: i - index + 1, value}); +} +if (matches.length > 1 && matches[matches.length - 1].value === target) { + for (const match of matches.slice(0, -1).reverse()) { + args.splice(match.index, match.count); + } +} +` + +func redirectJestCoverage(directory, coverage string, env map[string]string) error { + target, err := json.Marshal(coverage) + if err != nil { + return err + } + path := filepath.Join(directory, "coverage-arguments.cjs") + if err := os.WriteFile(path, []byte(strings.Replace(coverageArguments, "%s", string(target), 1)), 0600); err != nil { + return err + } + env["NODE_OPTIONS"] = "--require " + strconv.Quote(path) + " " + env["NODE_OPTIONS"] + return nil +} diff --git a/internal/testdrive/jest_coverage_integration_test.go b/internal/testdrive/jest_coverage_integration_test.go new file mode 100644 index 00000000..de009c7e --- /dev/null +++ b/internal/testdrive/jest_coverage_integration_test.go @@ -0,0 +1,140 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive_test + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// Exercise real Jest CLI parsing, including an existing CLI threshold, and +// verify that probes preserve coverage telemetry while bypassing suite quotas. +func TestJestCoverageProbeValidation(t *testing.T) { + requireNPMIntegration(t) + ctx, cancel := context.WithTimeout(t.Context(), fixtureTimeout) + defer cancel() + binary := filepath.Join(t.TempDir(), "ddtest") + integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + root, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + integrationCommand(t, ctx, root, nil, "git", "init", "-q") + integrationFile(t, root, "package.json", `{"scripts":{"test":"jest --runInBand"},"devDependencies":{"jest":"`+jestVersion+`"}}`) + integrationFile(t, root, "sum.js", `module.exports = (a, b) => a + b;`) + integrationFile(t, root, "sum.test.js", `test('adds', () => expect(require('./sum')(1, 2)).toBe(3));`) + config := `module.exports = {watchman: false, collectCoverage: true, collectCoverageFrom: ['sum.js'], coverageThreshold: {global: {lines: 100, statements: 100}}};` + integrationFile(t, root, "jest.config.js", config) + integrationCommand(t, ctx, root, nil, "npm", "install", "--no-audit", "--no-fund") + for _, mode := range []string{"default", "script", "cli"} { + t.Run(mode, func(t *testing.T) { + threshold := "100" + if mode == "script" { + threshold = "101" + } + script := "jest --runInBand" + if mode == "script" { + script += " --coverageDirectory script-coverage" + } + integrationFile(t, root, "package.json", `{"scripts":{"test":"`+script+`"},"devDependencies":{"jest":"`+jestVersion+`"}}`) + command := `npm test -- --coverageThreshold='{"global":{"lines":` + threshold + `}}'` + // Preserve customer coverage while redirecting options declared inside + // a package script or supplied explicitly on the command line. + if mode != "default" { + integrationFile(t, root, "coverage/customer.txt", "keep this coverage") + } + if mode == "cli" { + command += " --coverage-directory=explicit-coverage" + } + cmd := exec.CommandContext(ctx, binary, "testdrive", "--framework", "jest", "--yes", "--command", command) + cmd.Dir = root + cmd.Env = append(os.Environ(), "PWD="+root) + output, runErr := cmd.CombinedOutput() + require.Contains(t, string(output), "Keep this JSON report") + if threshold == "101" { + require.Contains(t, string(output), "Jest baseline command output:") + require.Contains(t, string(output), "Jest reporting-only command output:") + require.Contains(t, string(output), "Bounded diagnostic: .testoptimization/testdrive.json") + } + require.NotContains(t, string(output), "probe-fail-control command output:") + require.NotContains(t, string(output), "Full test output:") + data, err := os.ReadFile(filepath.Join(root, ".testoptimization", "testdrive.json")) + require.NoError(t, err) + var report struct { + Success bool `json:"success"` + Compatibility struct { + Status string `json:"status"` + } `json:"compatibility"` + Features []struct { + Status string `json:"status"` + } `json:"features"` + Runs []struct { + Name string `json:"name"` + Command string `json:"command"` + ExitCode int `json:"exit_code"` + ProbeMode string `json:"probe_mode"` + ThresholdsDisabled bool `json:"probe_coverage_thresholds_disabled"` + Diagnostic string `json:"diagnostic"` + } `json:"runs"` + } + require.NoError(t, json.Unmarshal(data, &report)) + for _, run := range report.Runs { + if run.Diagnostic != "" { + t.Log(run.Name, run.Diagnostic) + } + } + require.NoError(t, runErr, string(data)) + require.True(t, report.Success, string(data)) + require.Equal(t, "compatible", report.Compatibility.Status) + require.Len(t, report.Features, 6) + for _, feature := range report.Features { + require.Equal(t, "passed", feature.Status) + } + require.Len(t, report.Runs, 10) + for _, run := range report.Runs { + require.Equal(t, run.ProbeMode != "", run.ThresholdsDisabled) + if run.ProbeMode == "" { + require.NotContains(t, run.Command, "--coverageThreshold={}") + if threshold == "101" { + require.Equal(t, 1, run.ExitCode) + require.Contains(t, run.Diagnostic, "threshold (101%)") + } else { + require.Zero(t, run.ExitCode) + require.Empty(t, run.Diagnostic) + } + } + } + for _, name := range []string{"script-coverage", "explicit-coverage"} { + _, err := os.Stat(filepath.Join(root, name)) + require.True(t, os.IsNotExist(err), name) + } + if mode == "default" { + _, err := os.Stat(filepath.Join(root, "coverage")) + require.True(t, os.IsNotExist(err)) + } else { + entries, err := os.ReadDir(filepath.Join(root, "coverage")) + require.NoError(t, err) + require.Len(t, entries, 1) + data, err := os.ReadFile(filepath.Join(root, "coverage/customer.txt")) + require.NoError(t, err) + require.Equal(t, "keep this coverage", string(data)) + } + entries, err := os.ReadDir(filepath.Join(root, ".testoptimization")) + require.NoError(t, err) + require.Len(t, entries, 2, "retain HTML and compact JSON reports") + probes, err := filepath.Glob(filepath.Join(root, "*ddtest*")) + require.NoError(t, err) + require.Empty(t, probes) + after, err := os.ReadFile(filepath.Join(root, "jest.config.js")) + require.NoError(t, err) + require.Equal(t, config, string(after)) + }) + } +} diff --git a/internal/testdrive/jest_features.go b/internal/testdrive/jest_features.go new file mode 100644 index 00000000..9997fbd3 --- /dev/null +++ b/internal/testdrive/jest_features.go @@ -0,0 +1,338 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/DataDog/ddtest/internal/testdrive/intake" +) + +const probeName = "ddtest validation probe" +const probeSource = `let ddtestAttempts = 0; +test('ddtest validation probe', () => { + ddtestAttempts++; + if (process.env.DDTEST_PROBE_MODE === 'fail' || + (process.env.DDTEST_PROBE_MODE === 'fail-once' && ddtestAttempts === 1)) { + throw new Error('ddtest intentional failure'); + } +}); +` + +var jestFeatures = []string{"auto-retries", "early-flake-detection", "skipping", "quarantine", "disabled", "attempt-to-fix"} + +func unavailableFeatures(result *validationResult, reason string) { + for _, feature := range jestFeatures { + result.Features = append(result.Features, featureResult{Name: feature, Status: "inconclusive", Reason: reason}) + } +} + +func createJestProbe(root string, baseline validationRun) (string, error) { + return createProbe(root, baseline, probeSource) +} + +func createProbe(root string, baseline validationRun, sourceText string) (string, error) { + if executedTests(baseline) == 0 { + return "", errors.New("no executed test is available to locate a probe under the existing configuration") + } + root, err := filepath.EvalSymlinks(root) + if err != nil { + return "", err + } + for _, test := range baseline.Tests { + if test.Status != "passed" && test.Status != "failed" { + continue + } + source, err := filepath.EvalSymlinks(test.File) + if err != nil { + continue + } + relative, err := filepath.Rel(root, source) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + continue + } + // Letters preserve patterns such as +([a-zA-Z]).server.test.tsx. + // Discovery still verifies the actual project accepts this candidate. + letters := []byte(rand.Text()) + for i, value := range letters { + letters[i] = 'a' + value%26 + } + probe, err := os.OpenFile(filepath.Join(filepath.Dir(source), "ddtest"+string(letters)+filepath.Base(source)), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return "", err + } + _, writeErr := probe.WriteString(sourceText) + closeErr := probe.Close() + if err := errors.Join(writeErr, closeErr); err != nil { + return "", errors.Join(err, os.Remove(probe.Name())) + } + return probe.Name(), nil + } + return "", errors.New("no writable test location inside this repository was found for the probe") +} + +func (t *Testdrive) runJestFeatures(ctx context.Context, output io.Writer, session *Session, preload string, result *validationResult) (runErr error) { + if result.Preflight == nil || len(result.Preflight.Projects) == 0 { + unavailableFeatures(result, "Effective Jest projects were not resolved; feature coverage is unvalidated.") + return nil + } + names := map[string]int{} + for _, project := range result.Preflight.Projects { + names[project.DisplayName.Name]++ + } + for i, project := range result.Preflight.Projects { + label := project.DisplayName.Name + if label == "" || names[label] > 1 { + label = fmt.Sprintf("project %d", i+1) + } + firstFeature, firstRun := len(result.Features), len(result.Runs) + runner := *t + if len(result.Preflight.Projects) > 1 && (project.DisplayName.Name == "" || names[project.DisplayName.Name] != 1) { + unavailableFeatures(result, "Cannot unambiguously select this Jest project: a unique displayName is required for per-project feature validation. The original configuration was preserved.") + } else { + if len(result.Preflight.Projects) > 1 { + var selected bool + runner.args, selected = projectArgs(t.command, t.args, project.DisplayName.Name) + if !selected { + result.ProjectChecks = append(result.ProjectChecks, projectProbeCheck{Project: label, Root: project.Root, Excluded: true}) + continue + } + } + check := projectProbeCheck{Project: label, Root: project.Root} + err := runner.runJestProjectFeatures(ctx, output, session, preload, fmt.Sprintf("project-%d", i+1), &check, result) + result.ProjectChecks = append(result.ProjectChecks, check) + if err != nil { + runErr = errors.Join(runErr, err) + } + } + for j := firstFeature; j < len(result.Features); j++ { + result.Features[j].Project = label + } + for j := firstRun; j < len(result.Runs); j++ { + result.Runs[j].Project = label + } + if runErr != nil { + return runErr + } + } + return nil +} + +func (t *Testdrive) runJestProjectFeatures(ctx context.Context, output io.Writer, session *Session, preload, prefix string, check *projectProbeCheck, result *validationResult) (runErr error) { + files, command, err := t.discoverJestTests(ctx, "") + check.DiscoveryCommand = command + if err != nil || len(files) == 0 { + unavailableFeatures(result, fmt.Sprintf("Could not discover tests in this Jest project: %v", err)) + return nil + } + baseline := validationRun{} + for _, path := range files { + baseline.Tests = append(baseline.Tests, jestTest{File: path, Status: "passed"}) + } + probePath, err := createJestProbe(t.repositoryRoot, baseline) + if err != nil { + unavailableFeatures(result, err.Error()) + return nil + } + defer func() { runErr = errors.Join(runErr, os.Remove(probePath)) }() + check.Probe, _ = filepath.Rel(t.repositoryRoot, probePath) + files, check.ProbeDiscoveryCommand, err = t.discoverJestTests(ctx, probePath) + if err != nil || len(files) != 1 || filepath.Clean(files[0]) != filepath.Clean(probePath) { + check.Diagnostic = fmt.Sprintf("Probe discovery returned %d paths instead of the requested probe.", len(files)) + if err != nil { + check.Diagnostic = reportText(err.Error()) + } + unavailableFeatures(result, "The temporary probe was not discovered by this project's original configuration; its features remain unvalidated. "+check.Diagnostic) + return nil + } + check.Discovered = true + run := func(name string, scenario intake.Scenario, mode string) (validationRun, error) { + value, err := t.runJest(ctx, output, session, preload, prefix+"/"+name, true, scenario, probePath, mode) + value.Name = name + if d := value.Skipping; d != nil { + repoPath, repoErr := filepath.Rel(t.repositoryRoot, probePath) + projectPath, projectErr := filepath.Rel(check.Root, probePath) + if repoErr == nil && projectErr == nil { + d.RepositoryPath, d.ProjectPath = filepath.ToSlash(repoPath), filepath.ToSlash(projectPath) + d.PathMismatch = scenario.SourceFile != d.RepositoryPath && scenario.SourceFile != d.ProjectPath + } + } + return value, err + } + return runProbeFeatures(run, result) +} + +func runProbeFeatures(run func(string, intake.Scenario, string) (validationRun, error), result *validationResult) error { + // Verify the probe in the actual project configuration before claiming anything + // about feature behavior. Never fall back to a clean, unrelated Jest config. + passControl, err := run("probe-pass-control", intake.Scenario{}, "pass") + result.Runs = append(result.Runs, passControl.summary()) + if err != nil { + return err + } + failControl, err := run("probe-fail-control", intake.Scenario{}, "fail") + result.Runs = append(result.Runs, failControl.summary()) + if err != nil { + return err + } + if !validProbeControl(passControl, "passed") || !validProbeControl(failControl, "failed") { + unavailableFeatures(result, "The temporary probe did not produce its expected passing and failing control results under this framework configuration. See the recorded probe commands, exit codes, and bounded failure diagnostics.") + return nil + } + identity := passControl.Facts.Tests[0] + for _, feature := range jestFeatures { + if feature == "skipping" && identity.SourceFile == "" { + result.Features = append(result.Features, evaluateFeature(feature, validationRun{}, identity)) + continue + } + mode := "pass" + switch feature { + case "auto-retries": + mode = "fail-once" + case "skipping", "quarantine", "disabled": + mode = "fail" + } + scenario := intake.Scenario{Feature: feature, Module: identity.Module, Suite: identity.Suite, Test: identity.Name, SourceFile: identity.SourceFile} + value, err := run(feature, scenario, mode) + result.Runs = append(result.Runs, value.summary()) + if err != nil { + return err + } + result.Features = append(result.Features, evaluateFeature(feature, value, identity)) + } + return nil +} + +func validProbeControl(run validationRun, status string) bool { + expectedExit := (status == "passed" && run.ExitCode == 0) || (status == "failed" && run.ExitCode > 0) + if !expectedExit || run.ResultError != "" || executedTests(run) != 1 || len(run.SuiteErrors) != 0 || len(run.Facts.Tests) != 1 || verifyJestTelemetry(run).Status != "" { + return false + } + for _, test := range run.Tests { + if test.Name == probeName && test.Status == status { + return status != "failed" || strings.Contains(test.Failure, "ddtest intentional failure") + } + } + return false +} + +func evaluateFeature(feature string, run validationRun, identity intake.Test) featureResult { + result := featureResult{Name: feature, Status: "failed", Reason: "The controlled run did not show the expected feature behavior; see the recorded scenario command, exit code, and counts."} + if feature == "skipping" && identity.SourceFile == "" { + result.Status = "inconclusive" + result.Reason = "The control did not report test.source.file; a repository-relative file is required to request suite skipping." + return result + } + if run.ResultError != "" || len(run.SuiteErrors) > 0 || len(run.Facts.ConfigurationErrors) > 0 { + result.Status = "inconclusive" + result.Reason = "The scenario could not collect usable runner results or reported setup/configuration errors." + return result + } + if feature == "skipping" && run.framework == "vitest" && run.Facts.SkippableRequests == 0 { + result.Status = "inconclusive" + result.Reason = "The Vitest tracer did not request skippable suites. Check this tracer release's Vitest Test Impact Analysis support; a successful reporting run does not prove skipping is supported. Real Datadog credentials are not required." + return result + } + if run.ExitCode != 0 { + if d := run.Skipping; d != nil && d.PathMismatch && d.SkippableRequests > 0 { + result.Reason = "The mock returned the control's test.source.file, but that path differs from both repository-relative and project-relative probe paths. The probe executed instead of skipping; inspect the recorded skipping path mismatch. Preserve the original framework configuration. Real Datadog credentials are not required." + } + return result + } + passed, failed, retries := 0, 0, 0 + quarantined, disabled, attemptToFix, skipped := false, false, false, false + for _, event := range run.Facts.Events { + tags := event.Tags + if isSkippedJestProbe(event, identity.Module, identity.Suite, identity.SourceFile) { + skipped = true + } + if tags["test.suite"] != identity.Suite { + continue + } + if event.Type != "test" || tags["test.name"] != identity.Name || tags["test.module"] != identity.Module { + continue + } + if tags["test.status"] == "pass" { + passed++ + } + if tags["test.status"] == "fail" { + failed++ + } + reason := map[string]string{"auto-retries": "auto_test_retry", "early-flake-detection": "early_flake_detection", "attempt-to-fix": "attempt_to_fix"}[feature] + if reason != "" && tags["test.is_retry"] == "true" && tags["test.retry_reason"] == reason { + retries++ + } + quarantined = quarantined || tags["test.test_management.is_quarantined"] == "true" + disabled = disabled || (tags["test.test_management.is_test_disabled"] == "true" && tags["test.status"] == "skip") + attemptToFix = attemptToFix || tags["test.test_management.is_attempt_to_fix"] == "true" + } + nativePass, nativeFail, nativeSkip := 0, 0, 0 + for _, test := range run.Tests { + if test.Name != identity.Name { + result.Status = "inconclusive" + result.Reason = "The scenario selected tests beyond the controlled probe." + return result + } + switch test.Status { + case "passed": + nativePass++ + case "failed": + nativeFail++ + case "pending", "skipped": + nativeSkip++ + } + } + // Vitest reports one final native result for retries; attempt telemetry must + // still prove that the probe ran multiple times for the expected reason. + nativeRepeated := nativePass >= 2 || (run.framework == "vitest" && nativePass == 1) + success := false + switch feature { + case "auto-retries": + success = passed > 0 && failed == 1 && retries > 0 && nativePass > 0 && nativeFail == 0 + case "early-flake-detection": + success = passed >= 2 && failed == 0 && retries > 0 && nativeRepeated && nativeFail == 0 + case "skipping": + success = skipped && executedTests(run) == 0 && run.Facts.TestEventCount == 0 + case "quarantine": + success = quarantined && failed == 1 && retries == 0 && nativePass == 1 && nativeFail == 0 + case "disabled": + success = disabled && executedTests(run) == 0 && nativeSkip == 1 + case "attempt-to-fix": + success = attemptToFix && passed >= 2 && failed == 0 && retries > 0 && nativeRepeated && nativeFail == 0 + } + if !success { + result.Reason = fmt.Sprintf("Controlled %s did not show the expected behavior: matching telemetry has %d passed, %d failed and %d retries with the expected reason (%d total test events); runner has %d passed, %d failed and %d skipped; exit %d.", + feature, passed, failed, retries, run.Facts.TestEventCount, nativePass, nativeFail, nativeSkip, run.ExitCode) + if feature == "auto-retries" { + result.Reason += " The fail-once probe requires exactly one failed telemetry event followed by a successful retry; the runner may report only the final passing result." + if failed != 1 { + result.Reason = fmt.Sprintf("Retry telemetry count mismatch: expected 1 failed event, received %d. ", failed) + result.Reason + } + } + } + if success { + result.Status = "passed" + result.Reason = map[string]string{ + "auto-retries": "The fail-once probe failed, retried with auto_test_retry evidence, and recovered to a successful command.", + "early-flake-detection": "The new passing probe ran repeatedly with early_flake_detection retry evidence.", + "skipping": "The normally failing probe suite was skipped by ITR; no test body executed and the command succeeded.", + "quarantine": "The normally failing probe still reported its failure and quarantine tag, while the command succeeded.", + "disabled": "The normally failing probe was reported disabled/skipped, with no executed test body and a successful command.", + "attempt-to-fix": "The marked passing probe ran repeatedly with attempt_to_fix retry evidence and a successful command.", + }[feature] + if feature == "skipping" && run.Skipping != nil && run.Skipping.SuiteNameChanged { + result.Reason += " The tracer reported a different test.suite for the skipped suite; both names are retained in the skipping diagnostics." + } + } + return result +} diff --git a/internal/testdrive/jest_preflight.go b/internal/testdrive/jest_preflight.go new file mode 100644 index 00000000..b88951fb --- /dev/null +++ b/internal/testdrive/jest_preflight.go @@ -0,0 +1,176 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/platform" + "github.com/kballard/go-shellquote" +) + +type jestProject struct { + ID string `json:"id,omitempty"` + DisplayName struct { + Name string `json:"name"` + } `json:"displayName"` + Runner string `json:"testRunner"` + Environment string `json:"testEnvironment"` + Root string `json:"rootDir"` +} + +type jestPreflight struct { + Verdict verdict `json:"verdict"` + Command string `json:"command"` + Node string `json:"node"` + Version string `json:"version"` + VitestProjects []vitestProject `json:"vitest_projects,omitempty"` + Projects []jestProject `json:"projects"` + Recommendation *tracerRecommendation `json:"recommendation,omitempty"` + RecommendationNotes []string `json:"recommendation_notes,omitempty"` +} + +func (t *Testdrive) checkJestPreflight(ctx context.Context, output io.Writer, result *validationResult) error { + args := appendJestArgs(t.command, t.args, "--showConfig") + check := &jestPreflight{Node: t.nodeVersion(), Command: shellquote.Join(append([]string{t.command}, args...)...), Verdict: verdict{Status: "inconclusive", Reason: "Effective Jest configuration was not resolved."}} + result.Preflight = check + _, _ = fmt.Fprintf(output, "Inspecting Jest configuration: %s\n", check.Command) + // Loading Jest configuration may execute project JavaScript. This belongs in + // the approved execution phase, never detection or preview. Disable uploads. + env := testEnvironment("http://127.0.0.1:1", "preflight") + env["NODE_OPTIONS"] = stripDatadogNodeOptions(os.Getenv("NODE_OPTIONS")) + env["DD_CIVISIBILITY_ENABLED"] = "false" + env["DD_TRACE_ENABLED"] = "false" + data, err := t.executor.CombinedOutput(ctx, t.command, args, env) + if err != nil { + return fmt.Errorf("inspect Jest configuration: %w; verify --command includes the repository's Jest config and required setup", err) + } + var config struct { + Version string `json:"version"` + Configs []jestProject `json:"configs"` + } + // npm/yarn may print a script banner before Jest's JSON. + decoded := false + for i, b := range data { + if b == '{' && json.NewDecoder(strings.NewReader(string(data[i:]))).Decode(&config) == nil && config.Version != "" && len(config.Configs) > 0 { + decoded = true + break + } + } + if !decoded { + return fmt.Errorf("could not read Jest --showConfig; pass the actual Jest command including its config with --command") + } + check.Version = config.Version + check.Projects = config.Configs + installer, ok := t.platform.(*platform.JavaScript) + if !ok { + return fmt.Errorf("jest preflight requires the JavaScript tracer selector") + } + selection, err := installer.ResolveTestdriveTracer(ctx, platform.TracerOptions{Version: t.tracerVersion, Command: t.command, Args: t.args}) + result.Selection = &selection + if err != nil { + check.Verdict.Reason = "Jest configuration resolved, but tracer selection could not be resolved: " + reportText(err.Error()) + return err + } + result.Tracer = "dd-trace@" + selection.Version + result.TracerSource = selection.Source + if selection.Source == "project" { + _, _ = fmt.Fprintln(output, "Using the existing project tracer; --tracer-version applies only when it is absent.") + } + check.Verdict = checkJestSupport(*check, selection) + if result.CIRuntime != nil { + result.CISelection = compareCISelection(*result.CIRuntime, selection.Version) + } + _, _ = fmt.Fprintf(output, "Jest %s; Node %s; tracer %s (%s; requested %s)\nPreflight: %s — %s\n", check.Version, check.Node, result.Tracer, selection.Source, selection.Requested, check.Verdict.Status, check.Verdict.Reason) + t.recommendTracer(ctx, output, result) + if check.Verdict.Status == "incompatible" { + return fmt.Errorf("jest preflight: %s", check.Verdict.Reason) + } + if check.Verdict.Status != "compatible" && (!strings.HasPrefix(selection.Requested, "git:") || t.checkOnly) { + return fmt.Errorf("jest preflight is inconclusive: %s", check.Verdict.Reason) + } + return nil +} + +// Supported major lines mirror dd-trace-js's Jest instrumentation hooks. Keep +// this small table covered by boundary tests; unknown majors require review. +// Node engines alone do not describe framework/runner compatibility. +func checkJestSupport(check jestPreflight, selection platform.JSSelection) verdict { + var problems []string + for _, project := range check.Projects { + runner := filepath.ToSlash(project.Runner) + if !strings.Contains(runner, "/jest-circus/") && runner != "jest-circus/runner" { + problems = append(problems, "Jest requires the jest-circus runner for Test Optimization; add the matching jest-circus version and set testRunner to 'jest-circus/runner', or choose a supported Jest upgrade") + } + } + major, minor, ok := releaseParts(check.Version) + tracerMajor, _, known := releaseParts(selection.Version) + if known && (tracerMajor == 5 || tracerMajor == 6) && ok { + minimum := 24 + if tracerMajor == 6 { + minimum = 28 + } + if major < minimum || (major == 24 && minor < 8) { + problems = append(problems, fmt.Sprintf("dd-trace %d requires Jest >=%s; detected %s. Select a supported tracer with --tracer-version or review a Jest upgrade", tracerMajor, map[int]string{5: "24.8", 6: "28"}[tracerMajor], check.Version)) + } + } + if len(problems) > 0 { + return verdict{Status: "incompatible", Reason: strings.Join(problems, ". ")} + } + if !ok || !known || (tracerMajor != 5 && tracerMajor != 6) { + return verdict{Status: "inconclusive", Reason: "No verified Jest compatibility rule for this tracer/framework version; source builds are checked again after installation."} + } + status, reason := onboard.CompareNodeRequirement(check.Node, selection.Node) + if status != "compatible" { + return verdict{Status: status, Reason: reason} + } + return verdict{Status: "compatible", Reason: "Jest version, Circus runner, and local Node satisfy the known prerequisites. Paired execution and feature checks are still required."} +} + +func releaseParts(version string) (int, int, bool) { + parts := strings.Split(strings.TrimPrefix(version, "v"), ".") + if len(parts) != 3 || strings.ContainsAny(version, "-+") { + return 0, 0, false + } + major, e1 := strconv.Atoi(parts[0]) + minor, e2 := strconv.Atoi(parts[1]) + _, e3 := strconv.Atoi(parts[2]) + return major, minor, e1 == nil && e2 == nil && e3 == nil +} + +func verifyInstalledSelection(preload string, result *validationResult) error { + data, err := os.ReadFile(filepath.Join(filepath.Dir(filepath.Dir(preload)), "package.json")) + if err != nil { + return err + } + var manifest struct { + Version string `json:"version"` + Engines struct { + Node string `json:"node"` + } `json:"engines"` + } + if err = json.Unmarshal(data, &manifest); err != nil { + return err + } + if result.Selection.Version != "" && manifest.Version != result.Selection.Version { + return fmt.Errorf("installed dd-trace %s differs from preflight selection %s", manifest.Version, result.Selection.Version) + } + result.Selection.Version = manifest.Version + result.Selection.Node = manifest.Engines.Node + result.Tracer = "dd-trace@" + manifest.Version + if result.CIRuntime != nil { + result.CISelection = compareCISelection(*result.CIRuntime, manifest.Version) + } + return nil +} diff --git a/internal/testdrive/jest_preflight_test.go b/internal/testdrive/jest_preflight_test.go new file mode 100644 index 00000000..0e62db49 --- /dev/null +++ b/internal/testdrive/jest_preflight_test.go @@ -0,0 +1,169 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/platform" + "github.com/stretchr/testify/require" +) + +func TestJestPrerequisitesCoverLuxonAndTSyringe(t *testing.T) { + for _, tc := range []struct{ jest, runner, tracer, node, engine, status string }{ + {"24.9.0", "/node_modules/jest-jasmine2/build/index.js", "6.16.0", "24.14.1", ">=22", "incompatible"}, + {"24.9.0", "/node_modules/jest-circus/runner.js", "6.16.0", "24.14.1", ">=22", "incompatible"}, + {"24.9.0", "/node_modules/jest-circus/runner.js", "5.128.0", "24.14.1", ">=18", "compatible"}, + {"24.7.0", "jest-circus/runner", "5.128.0", "24.14.1", ">=18", "incompatible"}, + {"24.8.0", "jest-circus/runner", "5.128.0", "18.0.0", ">=18", "compatible"}, + {"27.5.1", "jest-circus/runner", "6.16.0", "24.14.1", ">=22", "incompatible"}, + {"28.0.0", "jest-circus/runner", "6.16.0", "24.14.1", ">=22", "compatible"}, + {"30.2.0", "jest-circus/runner", "6.16.0", "20.20.1", ">=22", "incompatible"}, + {"30.2.0", "jest-circus/runner", "6.16.0", "24.14.1", ">=22", "compatible"}, + {"30.2.0", "jest-circus/runner", "7.0.0", "24.14.1", ">=22", "inconclusive"}, + } { + result := checkJestSupport(jestPreflight{Version: tc.jest, Node: tc.node, Projects: []jestProject{{Runner: tc.runner}}}, platform.JSSelection{Version: tc.tracer, Node: tc.engine}) + require.Equal(t, tc.status, result.Status, "%+v: %s", tc, result.Reason) + } +} + +type preflightExecutor struct { + calls int + args []string + config string +} + +func (e *preflightExecutor) CombinedOutput(_ context.Context, _ string, args []string, _ map[string]string) ([]byte, error) { + e.calls++ + e.args = args + return []byte("package script banner\n" + e.config), nil +} + +func TestPreflightStopsUnsupportedJestBeforeInstallOrTests(t *testing.T) { + for _, runner := range []string{"jest-jasmine2/build/index.js", "jest-circus/runner.js"} { + t.Run(runner, func(t *testing.T) { + run := preflightFixture(t, "24.9.0", runner) + var output bytes.Buffer + require.ErrorContains(t, run.Run(t.Context(), &output), "dd-trace 6 requires Jest >=28") + require.Equal(t, 1, run.executor.(*preflightExecutor).calls) + data, err := os.ReadFile(validationPath(run.repositoryRoot)) + require.NoError(t, err) + var report validationResult + require.NoError(t, json.Unmarshal(data, &report)) + require.Equal(t, "incompatible", report.Preflight.Verdict.Status) + require.Empty(t, report.Runs) + require.False(t, report.Success) + require.Equal(t, "project", report.Selection.Source) + require.Equal(t, "6.16.0", report.Selection.Version) + if runner == "jest-circus/runner.js" { + require.Equal(t, "5.128.0", report.Preflight.Recommendation.Selection.Version) + require.Contains(t, output.String(), "--tracer-version 5.128.0") + } + }) + } +} + +func preflightFixture(t *testing.T, version, runner string) *Testdrive { + t.Helper() + if _, err := exec.LookPath("node"); err != nil { + t.Skip("Node is required to exercise project tracer resolution") + } + root := t.TempDir() + writeJestManifest(t, root) + t.Chdir(root) + tracerRoot := filepath.Join(root, "node_modules/dd-trace") + require.NoError(t, os.MkdirAll(filepath.Join(tracerRoot, "ci"), 0755)) + requireWriteFile(t, filepath.Join(tracerRoot, "ci/init.js"), "") + requireWriteFile(t, filepath.Join(tracerRoot, "package.json"), `{"version":"6.16.0","engines":{"node":">=22"}}`) + run, err := Prepare("latest-node18") + require.NoError(t, err) + run.command = "npx" + run.args = []string{"jest", "--config", "test/jest.config.js"} + data, err := json.Marshal(map[string]any{"version": version, "configs": []map[string]string{{"testRunner": "/node_modules/" + runner, "testEnvironment": "/jest-environment-node/build/index.js", "rootDir": root}}}) + require.NoError(t, err) + run.executor = &preflightExecutor{config: string(data)} + run.nodeVersion = func() string { return "24.14.1" } + run.resolveJSTracer = func(_ context.Context, major string) (platform.JSSelection, error) { + if major == "5" { + return platform.JSSelection{Version: "5.128.0", Node: ">=18"}, nil + } + return platform.JSSelection{Version: "6.17.0", Node: ">=22"}, nil + } + return run +} + +func TestCheckOnlyReportsNoTestExecutionAndProjectPrecedence(t *testing.T) { + run := preflightFixture(t, "30.2.0", "jest-circus/runner.js") + run.checkOnly = true + require.NoError(t, run.Run(t.Context(), &bytes.Buffer{})) + e := run.executor.(*preflightExecutor) + require.Equal(t, 1, e.calls) + require.Equal(t, []string{"jest", "--config", "test/jest.config.js", "--showConfig"}, e.args) + data, err := os.ReadFile(validationPath(run.repositoryRoot)) + require.NoError(t, err) + var report validationResult + require.NoError(t, json.Unmarshal(data, &report)) + require.True(t, report.ChecksPassed) + require.False(t, report.Success) + require.False(t, report.LocalSuccess) + require.Equal(t, "not exercised", report.Compatibility.Status) + require.Equal(t, "not exercised", report.CIExecution.Status) + require.Empty(t, report.Runs) + require.Equal(t, "latest-node18", report.Selection.Requested) + require.Equal(t, "6.16.0", report.Selection.Version) + require.Equal(t, "project", report.Selection.Source) +} + +func TestTracerMismatchCannotClaimOverallSuccess(t *testing.T) { + for _, tc := range []struct{ version, status, want string }{ + {"5.128.0", "compatible", "compatible"}, {"5.127.0", "compatible", "incompatible"}, {"", "inconclusive", "inconclusive"}, + } { + check := onboard.RuntimeCheck{Status: tc.status, Jobs: []onboard.RuntimeFinding{{Status: tc.status}}} + if tc.version != "" { + check.Jobs[0].Tracer = "dd-trace@" + tc.version + } + agreement := compareCISelection(check, "5.128.0") + require.Equal(t, tc.want, agreement.Status) + root := t.TempDir() + err := finishValidation(&bytes.Buffer{}, root, validationResult{Compatibility: verdict{Status: "compatible"}, CIRuntime: &check, CISelection: agreement}) + if tc.want == "compatible" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + data, err := os.ReadFile(validationPath(root)) + require.NoError(t, err) + var report validationResult + require.NoError(t, json.Unmarshal(data, &report)) + require.True(t, report.LocalSuccess) + require.Equal(t, tc.want == "compatible", report.Success) + } +} + +func TestJestInstallsExactPreflightSelection(t *testing.T) { + run := preparedTestdrive(t) + run.tracerVersion = "latest-node18" + run.preflight = func(_ context.Context, _ io.Writer, result *validationResult) error { + result.Selection = &platform.JSSelection{Requested: "latest-node18", Version: "5.128.0", Source: "fallback"} + return nil + } + installer := &fakeTracer{err: errors.New("installation stopped for assertion")} + run.platform = installer + require.ErrorContains(t, run.Run(t.Context(), &bytes.Buffer{}), "installation stopped for assertion") + require.Equal(t, "5.128.0", installer.options.Version) + require.Equal(t, run.command, installer.options.Command) + require.Equal(t, run.args, installer.options.Args) + require.NoDirExists(t, installer.options.Directory) +} diff --git a/internal/testdrive/jest_projects.go b/internal/testdrive/jest_projects.go new file mode 100644 index 00000000..e04f8f88 --- /dev/null +++ b/internal/testdrive/jest_projects.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + + "github.com/kballard/go-shellquote" +) + +type projectProbeCheck struct { + Project string `json:"project"` + Root string `json:"root"` + DiscoveryCommand string `json:"discovery_command"` + ProbeDiscoveryCommand string `json:"probe_discovery_command,omitempty"` + Probe string `json:"probe,omitempty"` + Diagnostic string `json:"diagnostic,omitempty"` + Discovered bool `json:"discovered"` + Excluded bool `json:"excluded,omitempty"` +} + +// Replace explicit project selectors only after preserving their scope. Jest +// accumulates repeated --selectProjects flags, which would run extra projects. +func projectArgs(command string, args []string, name string) ([]string, bool) { + var kept, selected, ignored []string + separator := jestSeparator(command, args) + for i := 0; i < len(args); i++ { + if i == separator { + kept = append(kept, args[i:]...) + break + } + flag, value, equals := strings.Cut(args[i], "=") + var target *[]string + switch flag { + case "--selectProjects", "--select-projects": + target = &selected + case "--ignoreProjects", "--ignore-projects": + target = &ignored + default: + kept = append(kept, args[i]) + continue + } + if equals { + *target = append(*target, value) + } else { + for i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") { + i++ + *target = append(*target, args[i]) + } + } + } + if (len(selected) > 0 && !slices.Contains(selected, name)) || slices.Contains(ignored, name) { + return nil, false + } + return appendJestArgs(command, kept, "--selectProjects", name), true +} + +func (t *Testdrive) discoverJestTests(ctx context.Context, probe string) ([]string, string, error) { + args := appendJestArgs(t.command, t.args, "--listTests", "--json") + if probe != "" { + args = withJestProbe(t.command, args, probe) + } + command := shellquote.Join(append([]string{t.command}, args...)...) + env := testEnvironment("http://127.0.0.1:1", "project-discovery") + env["NODE_OPTIONS"] = stripDatadogNodeOptions(os.Getenv("NODE_OPTIONS")) + env["DD_CIVISIBILITY_ENABLED"] = "false" + env["DD_TRACE_ENABLED"] = "false" + if probe != "" { + env["DDTEST_JEST_PROBE"] = "1" + } + data, err := t.executor.CombinedOutput(ctx, t.command, args, env) + if err != nil { + return nil, command, fmt.Errorf("jest --listTests failed: %w; %s", err, commandDiagnostic(data)) + } + // Package managers may print banners before the JSON array. + for i, value := range data { + var paths []string + if value == '[' && json.NewDecoder(strings.NewReader(string(data[i:]))).Decode(&paths) == nil { + slices.Sort(paths) + return slices.Compact(paths), command, nil + } + } + return nil, command, fmt.Errorf("jest --listTests did not return a JSON path array") +} diff --git a/internal/testdrive/jest_projects_integration_test.go b/internal/testdrive/jest_projects_integration_test.go new file mode 100644 index 00000000..f80e6f12 --- /dev/null +++ b/internal/testdrive/jest_projects_integration_test.go @@ -0,0 +1,114 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive_test + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// Two projects share a TS file but have different environments and match rules. +// Pin a tracer whose skipped suite names differ from executed suite names. +// Skipping must use source paths and retain that reporting difference. +func TestJestProjectsAndSkippingPathDiagnostics(t *testing.T) { + requireNPMIntegration(t) + ctx, cancel := context.WithTimeout(t.Context(), fixtureTimeout) + defer cancel() + binary := filepath.Join(t.TempDir(), "ddtest") + integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + root, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + integrationCommand(t, ctx, root, nil, "git", "init", "-q") + integrationFile(t, root, "package.json", `{"private":true,"devDependencies":{"jest":"30.4.2","jest-environment-jsdom":"30.4.1","@swc/jest":"0.2.39","dd-trace":"6.17.0"}}`) + integrationFile(t, root, "src/useForm.server.test.ts", `test('works', () => expect(1).toBe(1));`) + config := `module.exports = {projects: [ + {displayName:'Web',rootDir:'.',roots:['/src'],testMatch:['**/*.test.ts'],transform:{'^.+\\.tsx?$':'@swc/jest'},testEnvironment:'jsdom'}, + {displayName:'Server',rootDir:'.',roots:['/src'],testMatch:['**/+([a-zA-Z]).server.test.ts'],transform:{'^.+\\.tsx?$':'@swc/jest'},testEnvironment:'node'} +]};` + integrationFile(t, root, "scripts/jest/jest.config.js", config) + integrationCommand(t, ctx, root, nil, "npm", "install", "--no-audit", "--no-fund") + for _, aligned := range []bool{false, true} { + command := "./node_modules/.bin/jest --config scripts/jest/jest.config.js --runInBand --watchman=false" + if aligned { + // Diagnostic positive control only. ddtest must not change this itself. + command += " --rootDir ." + } + cmd := exec.CommandContext(ctx, binary, "testdrive", "--framework", "jest", "--yes", "--command", command) + cmd.Dir, cmd.Env = root, append(os.Environ(), "PWD="+root) + output, runErr := cmd.CombinedOutput() + data, err := os.ReadFile(filepath.Join(root, ".testoptimization", "testdrive.json")) + require.NoError(t, err, string(output)) + var report struct { + Success bool `json:"success"` + Features []struct{ Name, Status, Project, Reason string } `json:"features"` + Projects []struct { + Project string + Discovered bool + } `json:"project_checks"` + Runs []struct { + Name, Project, Command string + Skipping *struct { + SettingsRequests int `json:"settings_requests"` + SkippableRequests int `json:"skippable_requests"` + PathMismatch bool `json:"suite_path_mismatch"` + SuiteNameChanged bool `json:"suite_name_changed"` + SkippedByITR bool `json:"skipped_by_itr"` + SourceFile string `json:"source_file"` + ReturnedSuite string `json:"returned_suite"` + ExecutedTests int `json:"executed_tests"` + } `json:"skipping"` + } `json:"runs"` + } + require.NoError(t, json.Unmarshal(data, &report)) + require.True(t, report.Success, string(data)) + require.NoError(t, runErr, string(data)) + require.Len(t, report.Projects, 2, string(data)) + for _, project := range report.Projects { + require.True(t, project.Discovered, string(data)) + } + require.Len(t, report.Features, 12, string(data)) + for _, feature := range report.Features { + if feature.Name == "skipping" && !aligned { + require.Contains(t, feature.Reason, "different test.suite") + } + require.Equal(t, "passed", feature.Status, string(data)) + require.Contains(t, []string{"Web", "Server"}, feature.Project) + } + require.Len(t, report.Runs, 18) + for _, run := range report.Runs { + if run.Name == "skipping" { + require.NotNil(t, run.Skipping) + require.Positive(t, run.Skipping.SettingsRequests) + require.Positive(t, run.Skipping.SkippableRequests) + require.False(t, run.Skipping.PathMismatch) + require.Equal(t, !aligned, run.Skipping.SuiteNameChanged) + require.True(t, run.Skipping.SkippedByITR) + require.Equal(t, run.Skipping.SourceFile, run.Skipping.ReturnedSuite) + require.Contains(t, run.Skipping.ReturnedSuite, "src/") + require.NotContains(t, run.Skipping.ReturnedSuite, "../") + require.Zero(t, run.Skipping.ExecutedTests) + } + if run.Project != "" { + require.Contains(t, run.Command, "--selectProjects "+run.Project) + } + } + entries, err := os.ReadDir(filepath.Join(root, ".testoptimization")) + require.NoError(t, err) + require.Len(t, entries, 1) + probes, err := filepath.Glob(filepath.Join(root, "src/ddtest*")) + require.NoError(t, err) + require.Empty(t, probes) + after, err := os.ReadFile(filepath.Join(root, "scripts/jest/jest.config.js")) + require.NoError(t, err) + require.Equal(t, config, string(after)) + } +} diff --git a/internal/testdrive/jest_projects_test.go b/internal/testdrive/jest_projects_test.go new file mode 100644 index 00000000..163c81e2 --- /dev/null +++ b/internal/testdrive/jest_projects_test.go @@ -0,0 +1,90 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "regexp" + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +type discoveryExecutor func([]string) ([]byte, error) + +func (f discoveryExecutor) CombinedOutput(_ context.Context, _ string, args []string, _ map[string]string) ([]byte, error) { + return f(args) +} + +func TestProjectProbeKeepsLettersOnlyServerPattern(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "useForm.server.test.tsx") + requireWriteFile(t, source, "original") + path, err := createJestProbe(root, validationRun{Tests: []jestTest{{File: source, Status: "passed"}}}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.Remove(path)) }) + require.Regexp(t, regexp.MustCompile(`^[a-zA-Z]+\.server\.test\.tsx$`), filepath.Base(path)) +} + +func TestProjectSelectionPreservesCommandScope(t *testing.T) { + args := []string{"test", "--", "--config", "scripts/jest.js", "--selectProjects", "Web", "Server", "--ignoreProjects=Server", "--ci"} + selected, ok := projectArgs("npm", args, "Web") + require.True(t, ok) + require.Equal(t, []string{"test", "--", "--config", "scripts/jest.js", "--ci", "--selectProjects", "Web"}, selected) + _, ok = projectArgs("npm", args, "Server") + require.False(t, ok) + _, ok = projectArgs("npm", args, "Other") + require.False(t, ok) + require.Contains(t, args, "--ignoreProjects=Server", "must not mutate the full-suite command") +} + +func TestEveryUnselectableProjectIsUnvalidated(t *testing.T) { + for _, duplicateName := range []string{"", "same"} { + run := preparedTestdrive(t) + project := jestProject{} + project.DisplayName.Name = duplicateName + result := validationResult{Preflight: &jestPreflight{Projects: []jestProject{project, project}}} + require.NoError(t, run.runJestFeatures(t.Context(), &bytes.Buffer{}, nil, "", &result)) + require.Len(t, result.Features, 12) + for i, feature := range result.Features { + require.Equal(t, "inconclusive", feature.Status) + require.Equal(t, []string{"project 1", "project 2"}[i/6], feature.Project) + } + } +} + +func TestUndiscoveredProbeCannotValidateWrongProject(t *testing.T) { + run := preparedTestdrive(t) + source := filepath.Join(run.repositoryRoot, "onlyExactName.test.js") + requireWriteFile(t, source, "original") + run.executor = discoveryExecutor(func(args []string) ([]byte, error) { + if slices.Contains(args, "--runTestsByPath") { + return []byte("[]"), nil + } + return json.Marshal([]string{source}) + }) + result := validationResult{Preflight: &jestPreflight{Projects: []jestProject{{Root: run.repositoryRoot}}}} + require.NoError(t, run.runJestFeatures(t.Context(), &bytes.Buffer{}, nil, "", &result)) + require.Len(t, result.Features, 6) + for _, feature := range result.Features { + require.Equal(t, "inconclusive", feature.Status) + require.Contains(t, feature.Reason, "not discovered") + } + require.Len(t, result.ProjectChecks, 1) + require.False(t, result.ProjectChecks[0].Discovered) + require.Contains(t, result.ProjectChecks[0].ProbeDiscoveryCommand, "--runTestsByPath") + files, err := filepath.Glob(filepath.Join(run.repositoryRoot, "ddtest*")) + require.NoError(t, err) + require.Empty(t, files) + data, err := os.ReadFile(source) + require.NoError(t, err) + require.Equal(t, "original", string(data)) +} diff --git a/internal/testdrive/jest_recommendation.go b/internal/testdrive/jest_recommendation.go new file mode 100644 index 00000000..a04714d9 --- /dev/null +++ b/internal/testdrive/jest_recommendation.go @@ -0,0 +1,132 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "context" + "fmt" + "io" + "slices" + "strings" + "time" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/platform" +) + +type tracerRecommendation struct { + Selection platform.JSSelection `json:"selection"` + Compatible []string `json:"compatible_ci_nodes,omitempty"` + Unsupported []string `json:"unsupported_ci_nodes,omitempty"` + Unresolved []string `json:"unresolved_ci_nodes,omitempty"` +} + +// Prefer coverage of known CI runtimes, then the newest verified major. Never +// install a candidate or change the selected/project tracer based on this advice. +func recommendJavaScriptTracer(ctx context.Context, check jestPreflight, ci *onboard.RuntimeCheck, resolve func(context.Context, string) (platform.JSSelection, error), framework string) (*tracerRecommendation, []string) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var best *tracerRecommendation + var notes []string + for _, major := range []string{"6", "5"} { // Resolve current releases from the supported tracer lines; no release pins. + selected, err := resolve(ctx, major) + if err != nil { + notes = append(notes, fmt.Sprintf("Could not resolve dd-trace %s: %s", major, reportText(err.Error()))) + continue + } + support := checkJestSupport + if framework == "vitest" { + support = checkVitestSupport + } + if verdict := support(check, selected); verdict.Status != "compatible" { + continue + } + candidate := &tracerRecommendation{Selection: selected} + if ci != nil { + for _, job := range ci.Jobs { + node := job.Node + if job.NodeResolution != nil { + node = job.NodeResolution.Version + } + label := nodeScopeLabel(job.Node) + status, _ := onboard.CompareNodeRequirement(node, selected.Node) + switch status { + case "compatible": + candidate.Compatible = appendUnique(candidate.Compatible, label) + case "incompatible": + candidate.Unsupported = appendUnique(candidate.Unsupported, label) + default: + candidate.Unresolved = appendUnique(candidate.Unresolved, label) + } + } + // No candidate is useful for an entirely incompatible known CI matrix. + if len(candidate.Compatible) == 0 && len(candidate.Unsupported) > 0 { + continue + } + } + if best == nil || len(candidate.Compatible) > len(best.Compatible) { + best = candidate + } + } + if best == nil { + notes = append(notes, fmt.Sprintf("No verified tracer candidate satisfies the detected %s configuration, local Node and any known CI runtime. Review prerequisites; do not drop test coverage or guess older majors.", framework)) + } + return best, notes +} + +func appendUnique(values []string, value string) []string { + if !slices.Contains(values, value) { + return append(values, value) + } + return values +} + +func scopeList(values []string) string { + if len(values) == 0 { + return "none" + } + if len(values) > 8 { + return strings.Join(values[:8], ", ") + fmt.Sprintf(" and %d more (see ci_runtime.jobs)", len(values)-8) + } + return strings.Join(values, ", ") +} + +func nodeScopeLabel(node string) string { + if node == "" { + return "unknown" + } + value := []rune(node) + if len(value) > 80 { + return string(value[:80]) + "…" + } + return node +} + +// Recommend before excluding older CI runtimes, including when the remaining +// instrumented jobs already pass static validation. Never change dependencies. +func (t *Testdrive) recommendTracer(ctx context.Context, output io.Writer, result *validationResult) { + check, selection := result.Preflight, result.Selection + if t.resolveJSTracer == nil || strings.HasPrefix(selection.Requested, "git:") { + return + } + needed := check.Verdict.Status != "compatible" + if ci := result.CIRuntime; ci != nil { + needed = needed || (ci.Status != "compatible" && ci.Status != "not applicable") + for _, job := range ci.Jobs { + needed = needed || job.Code == "excluded_runtime" + } + } + if !needed { + return + } + check.Recommendation, check.RecommendationNotes = recommendJavaScriptTracer(ctx, *check, result.CIRuntime, t.resolveJSTracer, t.framework.Name()) + if rec := check.Recommendation; rec != nil { + _, _ = fmt.Fprintf(output, "Recommended candidate: dd-trace@%s (Node %s). CI Node compatible: %s; unsupported: %s; unresolved: %s.\nPrefer a candidate covering the whole existing matrix before excluding runtimes. Use --tracer-version %s and js-tracer-version: %s, then rerun --check-only before full validation. This is advisory; the current selection and project dependency are unchanged.\n", rec.Selection.Version, rec.Selection.Node, scopeList(rec.Compatible), scopeList(rec.Unsupported), scopeList(rec.Unresolved), rec.Selection.Version, rec.Selection.Version) + } + for _, note := range check.RecommendationNotes { + _, _ = fmt.Fprintln(output, "Tracer recommendation: "+note) + } +} diff --git a/internal/testdrive/jest_recommendation_test.go b/internal/testdrive/jest_recommendation_test.go new file mode 100644 index 00000000..2edcee3d --- /dev/null +++ b/internal/testdrive/jest_recommendation_test.go @@ -0,0 +1,115 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "context" + "errors" + "testing" + + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/platform" + "github.com/stretchr/testify/require" +) + +func TestTracerRecommendationMatchesFrameworkLocalNodeAndCIMatrix(t *testing.T) { + resolve := func(_ context.Context, major string) (platform.JSSelection, error) { + return map[string]platform.JSSelection{ + "5": {Version: "5.128.0", Node: ">=18"}, + "6": {Version: "6.17.0", Node: ">=22"}, + }[major], nil + } + for _, tc := range []struct { + name, jest, node, runner string + nodes []string + want string + }{ + {"i18next", "28.1.3", "24.14.1", "jest-circus/runner", []string{"14.x", "16.x", "18.x", "20.x", "20.x"}, "5.128.0"}, + {"newest breaks tie", "28.1.3", "24.14.1", "jest-circus/runner", []string{"22.x", "24.x"}, "6.17.0"}, + {"framework limits selection", "27.5.0", "24.14.1", "jest-circus/runner", []string{"24.x"}, "5.128.0"}, + {"local Node limits selection", "28.1.3", "20.10.0", "jest-circus/runner", nil, "5.128.0"}, + {"unsupported CI", "28.1.3", "24.14.1", "jest-circus/runner", []string{"14.x"}, ""}, + {"unsupported runner", "28.1.3", "24.14.1", "jest-jasmine2", []string{"24.x"}, ""}, + {"unresolved CI", "28.1.3", "24.14.1", "jest-circus/runner", []string{"lts/*"}, "6.17.0"}, + } { + t.Run(tc.name, func(t *testing.T) { + ci := &onboard.RuntimeCheck{} + for _, node := range tc.nodes { + ci.Jobs = append(ci.Jobs, onboard.RuntimeFinding{Node: node}) + } + check := jestPreflight{Version: tc.jest, Node: tc.node, Projects: []jestProject{{Runner: tc.runner}}} + got, notes := recommendJavaScriptTracer(t.Context(), check, ci, resolve, "jest") + if tc.want == "" { + require.Nil(t, got) + require.NotEmpty(t, notes) + return + } + require.Empty(t, notes) + require.Equal(t, tc.want, got.Selection.Version) + if tc.name == "i18next" { + require.Equal(t, []string{"18.x", "20.x"}, got.Compatible) + require.Equal(t, []string{"14.x", "16.x"}, got.Unsupported) + } + if tc.name == "unresolved CI" { + require.Empty(t, got.Compatible) + require.Equal(t, []string{"lts/*"}, got.Unresolved) + } + }) + } +} + +func TestTracerRecommendationReportsMetadataFailure(t *testing.T) { + got, notes := recommendJavaScriptTracer(t.Context(), jestPreflight{}, nil, func(context.Context, string) (platform.JSSelection, error) { + return platform.JSSelection{}, errors.New("registry unavailable") + }, "jest") + require.Nil(t, got) + require.Len(t, notes, 3) + require.Contains(t, notes[0], "registry unavailable") +} + +func TestVitestRecommendationCoversOlderCIWithoutChangingSelection(t *testing.T) { + for _, excluded := range []bool{false, true} { + drive := preparedTestdrive(t) + drive.framework = framework.NewVitest() + drive.resolveJSTracer = func(_ context.Context, major string) (platform.JSSelection, error) { + if major == "5" { + return platform.JSSelection{Version: "5.129.0", Node: ">=18"}, nil + } + return platform.JSSelection{Version: "6.18.0", Node: ">=22"}, nil + } + ci := &onboard.RuntimeCheck{Status: "incompatible", Jobs: []onboard.RuntimeFinding{{Node: "20.x"}, {Node: "22.x"}}} + if excluded { + ci.Status = "compatible" + ci.Jobs[0].Status = "excluded" + ci.Jobs[0].Code = "excluded_runtime" + } + result := validationResult{CIRuntime: ci, Selection: &platform.JSSelection{Version: "6.18.0", Source: "project"}, Preflight: &jestPreflight{Node: "24.14.1", Version: "4.0.1", VitestProjects: []vitestProject{{Pool: "forks"}}, Verdict: verdict{Status: "compatible"}}} + var output bytes.Buffer + drive.recommendTracer(t.Context(), &output, &result) + require.NotNil(t, result.Preflight.Recommendation) + require.Equal(t, "5.129.0", result.Preflight.Recommendation.Selection.Version) + require.Equal(t, []string{"20.x", "22.x"}, result.Preflight.Recommendation.Compatible) + require.Equal(t, "6.18.0", result.Selection.Version) + require.Equal(t, "project", result.Selection.Source) + require.Contains(t, output.String(), "before excluding runtimes") + } +} + +func TestVitestRecommendationRespectsFeaturePrerequisitesAndUnknownRuntimes(t *testing.T) { + check := jestPreflight{Node: "24.14.1", Version: "4.0.1", VitestProjects: []vitestProject{{Pool: "forks"}}} + ci := &onboard.RuntimeCheck{Jobs: []onboard.RuntimeFinding{{Node: "20"}, {Node: "22"}, {Node: "lts/*"}}} + got, _ := recommendJavaScriptTracer(t.Context(), check, ci, func(_ context.Context, major string) (platform.JSSelection, error) { + if major == "5" { + return platform.JSSelection{Version: "5.119.0", Node: ">=18"}, nil + } + return platform.JSSelection{Version: "6.18.0", Node: ">=22"}, nil + }, "vitest") + require.Equal(t, "6.18.0", got.Selection.Version, "older tracer lacks Vitest skipping") + require.Equal(t, []string{"20"}, got.Unsupported) + require.Equal(t, []string{"lts/*"}, got.Unresolved) +} diff --git a/internal/testdrive/jest_skipping.go b/internal/testdrive/jest_skipping.go new file mode 100644 index 00000000..942d8ded --- /dev/null +++ b/internal/testdrive/jest_skipping.go @@ -0,0 +1,65 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import "github.com/DataDog/ddtest/internal/testdrive/intake" + +// Retain only controlled identities and counts, never full requests or events. +type skippingDiagnostic struct { + SettingsRequests int `json:"settings_requests"` + SkippableRequests int `json:"skippable_requests"` + RequestedSuite string `json:"requested_probe_suite"` + SourceFile string `json:"source_file"` + ReturnedSuite string `json:"returned_suite,omitempty"` + ObservedSuite string `json:"observed_probe_suite,omitempty"` + ObservedSourceFile string `json:"observed_source_file,omitempty"` + SuiteNameChanged bool `json:"suite_name_changed"` + IdentityMatched bool `json:"observed_identity_matched"` + RepositoryPath string `json:"repository_relative_probe,omitempty"` + ProjectPath string `json:"project_relative_probe,omitempty"` + PathMismatch bool `json:"suite_path_mismatch"` + SkippedByITR bool `json:"skipped_by_itr"` + ExecutedTests int `json:"executed_tests"` + Expected string `json:"expected"` +} + +func diagnoseSkipping(run validationRun, scenario intake.Scenario) *skippingDiagnostic { + d := &skippingDiagnostic{SettingsRequests: run.Facts.SettingsRequests, SkippableRequests: run.Facts.SkippableRequests, + RequestedSuite: reportText(scenario.Suite), SourceFile: reportText(scenario.SourceFile), ExecutedTests: executedTests(run), + Expected: "Mock returns the control's test.source.file; ITR skips that file, no test body executes, and exit code is 0. Suite naming differences are retained separately. Real Datadog credentials are not required."} + if d.SkippableRequests > 0 { + d.ReturnedSuite = reportText(scenario.SourceFile) + } + for _, event := range run.Facts.Events { + tags := event.Tags + if event.Type == "test" && tags["test.name"] == scenario.Test && tags["test.module"] == scenario.Module { + d.ObservedSuite = reportText(tags["test.suite"]) + d.ObservedSourceFile = reportText(tags["test.source.file"]) + d.IdentityMatched = tags["test.suite"] == scenario.Suite + } + if isSkippedJestProbe(event, scenario.Module, scenario.Suite, scenario.SourceFile) { + d.SkippedByITR = true + d.ObservedSuite = reportText(tags["test.suite"]) + d.ObservedSourceFile = reportText(tags["test.source.file"]) + d.IdentityMatched = tags["test.suite"] == scenario.Suite + d.SuiteNameChanged = !d.IdentityMatched + } + } + return d +} + +func isSkippedJestProbe(event intake.Event, module, suite, sourceFile string) bool { + tags := event.Tags + if sourceFile == "" || event.Type != "test_suite_end" || tags["test.module"] != module || tags["test.status"] != "skip" || tags["test.skipped_by_itr"] != "true" { + return false + } + if observedSource := tags["test.source.file"]; observedSource != "" && observedSource != sourceFile { + return false + } + // Current dd-trace uses the returned file path as the skipped suite name. + // Also accept tracers that preserve the executed suite's reported identity. + return tags["test.suite"] == sourceFile || tags["test.suite"] == suite +} diff --git a/internal/testdrive/jest_test.go b/internal/testdrive/jest_test.go new file mode 100644 index 00000000..850403fa --- /dev/null +++ b/internal/testdrive/jest_test.go @@ -0,0 +1,396 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/kballard/go-shellquote" + "github.com/stretchr/testify/require" +) + +func testRun(status string, exit int) validationRun { + traceStatus := map[string]string{"passed": "pass", "failed": "fail"}[status] + return validationRun{ExitCode: exit, Tests: []jestTest{{File: "/project/test.js", Name: "works", Status: status}}, Facts: intake.Facts{ + TestEventCount: 1, Tests: []intake.Test{{Name: "works", SourceFile: "/project/test.js", Attempts: []intake.TestRun{{Status: traceStatus}}}}, + }} +} + +func TestCompareJestContract(t *testing.T) { + for _, status := range []string{"passed", "failed"} { + t.Run(status, func(t *testing.T) { + exit := 0 + if status == "failed" { + exit = 1 + } + baseline := testRun(status, exit) + require.Equal(t, "compatible", compareJest(withoutTelemetry(baseline), baseline).Status) + }) + } + baseline := withoutTelemetry(testRun("passed", 0)) + failed := testRun("failed", 1) + require.Equal(t, "suspected regression", compareJest(baseline, failed).Status) + failed.Tests[0].Failure = "expected true" + differentFailure := testRun("failed", 1) + differentFailure.Tests[0].Failure = "expected false" + require.Equal(t, "suspected regression", compareJest(withoutTelemetry(failed), differentFailure).Status) + missing := testRun("passed", 0) + missing.Tests = nil + require.Equal(t, "suspected regression", compareJest(baseline, missing).Status) + noTelemetry := testRun("passed", 0) + noTelemetry.Facts = intake.Facts{} + require.Equal(t, "inconclusive", compareJest(baseline, noTelemetry).Status) + wrongTelemetry := testRun("passed", 0) + wrongTelemetry.Facts.Tests[0].Name = "different test" + mismatch := compareJest(baseline, wrongTelemetry) + require.Equal(t, "inconclusive", mismatch.Status) + require.Equal(t, []string{ + `Missing telemetry: file "/project/test.js", test "works", status "pass" (count 1)`, + `Unexpected telemetry: file "/project/test.js", test "different test", status "pass" (count 1)`, + }, mismatch.Differences) + require.Equal(t, "inconclusive", compareJest(validationRun{}, validationRun{}).Status) + baseline.Tests = append(baseline.Tests, baseline.Tests[0]) + require.Equal(t, "suspected regression", compareJest(baseline, testRun("passed", 0)).Status) +} + +func TestJestJSONComparisonIgnoresTimingOrderAndStacks(t *testing.T) { + a, b := validationRun{}, validationRun{} + first := `{"testResults":[{"name":"/repo/a.test.js","assertionResults":[{"fullName":"suite first","status":"passed","duration":2},{"fullName":"suite second","status":"failed","failureMessages":["Error: expected 2\n at project.js:1"]}]}]}` + second := `{"testResults":[{"name":"/repo/a.test.js","assertionResults":[{"fullName":"suite second","status":"failed","failureMessages":["Error: expected 2\n at dd-trace.js:100\n at project.js:1"]},{"fullName":"suite first","status":"passed","duration":100}]}]}` + path := filepath.Join(t.TempDir(), "results.json") + require.NoError(t, os.WriteFile(path, []byte(first), 0600)) + readJestResults(path, &a) + require.NoError(t, os.WriteFile(path, []byte(second), 0600)) + readJestResults(path, &b) + require.Equal(t, outcomeKeys(a), outcomeKeys(b)) +} + +func TestScenarioEnvironmentsDisableUnrelatedBehavior(t *testing.T) { + for _, feature := range append([]string{""}, jestFeatures...) { + env := testEnvironment("http://127.0.0.1:1234", "session") + configureScenarioEnvironment(env, feature) + require.Equal(t, "false", env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"]) + require.Equal(t, "false", env["DD_TEST_FAILED_TEST_REPLAY_ENABLED"]) + require.Equal(t, feature == "auto-retries", env["DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"] == "true") + require.Equal(t, feature == "early-flake-detection", env["DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED"] == "true") + require.Equal(t, slices.Contains([]string{"quarantine", "disabled", "attempt-to-fix"}, feature), env["DD_TEST_MANAGEMENT_ENABLED"] == "true") + } +} + +func TestProbeUsesProjectTestLocationAndDoesNotOverwrite(t *testing.T) { + root := t.TempDir() + original := filepath.Join(root, "existing.test.js") + requireWriteFile(t, original, "original") + run := testRun("passed", 0) + run.Tests[0].File = original + path, err := createJestProbe(root, run) + require.NoError(t, err) + defer func() { require.NoError(t, os.Remove(path)) }() + require.NotEqual(t, original, path) + resolvedRoot, err := filepath.EvalSymlinks(root) + require.NoError(t, err) + require.Equal(t, resolvedRoot, filepath.Dir(path)) + contents, err := os.ReadFile(original) + require.NoError(t, err) + require.Equal(t, "original", string(contents)) + run.Tests[0].File = filepath.Join(t.TempDir(), "outside.test.js") + requireWriteFile(t, run.Tests[0].File, "original") + _, err = createJestProbe(root, run) + require.Error(t, err) +} + +func TestFeatureAssertionsRequireBehaviorAndTelemetry(t *testing.T) { + identity := intake.Test{Module: "jest", Suite: "probe.test.js", SourceFile: "probe.test.js", Name: probeName} + event := func(status, reason string, extra map[string]string) intake.Event { + tags := map[string]string{"test.module": "jest", "test.suite": "probe.test.js", "test.name": probeName, "test.status": status} + if reason != "" { + tags["test.is_retry"] = "true" + tags["test.retry_reason"] = reason + } + for key, value := range extra { + tags[key] = value + } + return intake.Event{Type: "test", Tags: tags} + } + cases := map[string][]intake.Event{ + "auto-retries": {event("fail", "", nil), event("pass", "auto_test_retry", nil)}, + "early-flake-detection": {event("pass", "", nil), event("pass", "early_flake_detection", nil)}, + "quarantine": {event("fail", "", map[string]string{"test.test_management.is_quarantined": "true"})}, + "disabled": {event("skip", "", map[string]string{"test.test_management.is_test_disabled": "true"})}, + "attempt-to-fix": {event("pass", "", map[string]string{"test.test_management.is_attempt_to_fix": "true"}), event("pass", "attempt_to_fix", nil)}, + "skipping": {{Type: "test_suite_end", Tags: map[string]string{"test.module": "jest", "test.suite": "probe.test.js", "test.status": "skip", "test.skipped_by_itr": "true"}}}, + } + for feature, events := range cases { + t.Run(feature, func(t *testing.T) { + run := validationRun{Facts: intake.Facts{Events: events}} + if feature != "skipping" { + run.Tests = []jestTest{{Name: probeName, Status: "passed"}} + if feature == "disabled" { + run.Tests[0].Status = "pending" + } + if feature == "early-flake-detection" || feature == "attempt-to-fix" { + run.Tests = append(run.Tests, run.Tests[0]) + } + } + require.Equal(t, "passed", evaluateFeature(feature, run, identity).Status) + run.ExitCode = 1 + require.Equal(t, "failed", evaluateFeature(feature, run, identity).Status) + require.Equal(t, "failed", evaluateFeature(feature, validationRun{}, identity).Status) + }) + } + run := validationRun{ResultError: "missing JSON"} + require.Equal(t, "inconclusive", evaluateFeature("auto-retries", run, identity).Status) +} + +func TestJestSkippingUsesSourceFileWithoutChangingOtherFeatureIdentities(t *testing.T) { + identity := intake.Test{Module: "jest", Suite: "../../src/probe.test.js", SourceFile: "src/probe.test.js", Name: probeName} + for _, suite := range []string{identity.SourceFile, identity.Suite} { + t.Run(suite, func(t *testing.T) { + tags := map[string]string{"test.module": "jest", "test.suite": suite, "test.status": "skip", "test.skipped_by_itr": "true"} + run := validationRun{Facts: intake.Facts{Events: []intake.Event{{Type: "test_suite_end", Tags: tags}}}} + require.Equal(t, "passed", evaluateFeature("skipping", run, identity).Status) + for _, entry := range []struct{ key, value string }{ + {"test.suite", "src/unrelated.test.js"}, + {"test.module", "other"}, + {"test.status", "pass"}, + {"test.skipped_by_itr", "false"}, + {"test.source.file", "src/unrelated.test.js"}, + } { + original := tags[entry.key] + tags[entry.key] = entry.value + require.Equal(t, "failed", evaluateFeature("skipping", run, identity).Status, entry.key) + tags[entry.key] = original + } + run.Facts.TestEventCount = 1 + require.Equal(t, "failed", evaluateFeature("skipping", run, identity).Status) + run.Facts.TestEventCount = 0 + run.Tests = []jestTest{{Name: probeName, Status: "passed"}} + require.Equal(t, "failed", evaluateFeature("skipping", run, identity).Status) + }) + } + missing := identity + missing.SourceFile = "" + require.Equal(t, "inconclusive", evaluateFeature("skipping", validationRun{}, missing).Status) + tags := map[string]string{"test.module": "jest", "test.suite": identity.Suite, "test.name": probeName, + "test.status": "skip", "test.test_management.is_test_disabled": "true"} + run := validationRun{Tests: []jestTest{{Name: probeName, Status: "pending"}}, Facts: intake.Facts{Events: []intake.Event{{Type: "test", Tags: tags}}}} + require.Equal(t, "passed", evaluateFeature("disabled", run, identity).Status) + tags["test.suite"] = identity.SourceFile + require.Equal(t, "failed", evaluateFeature("disabled", run, identity).Status) +} + +type jsonExecutor struct { + results []string + exits []error + calls int + envs []map[string]string + commands []string +} + +func (e *jsonExecutor) CombinedOutput(_ context.Context, command string, args []string, env map[string]string) ([]byte, error) { + index := slices.Index(args, "--outputFile") + if index < 0 { + return nil, errors.New("missing JSON output argument") + } + err := os.WriteFile(args[index+1], []byte(e.results[e.calls]), 0600) + if err != nil { + return nil, err + } + e.envs = append(e.envs, env) + e.commands = append(e.commands, shellquote.Join(append([]string{command}, args...)...)) + exit := e.exits[e.calls] + e.calls++ + return []byte("detailed test output"), exit +} + +func TestJestValidationRepeatsUnstablePairAndWritesJSON(t *testing.T) { + run := preparedTestdrive(t) + run.platform = &fakeTracer{preloadPath: "/trace/ci/init.js"} + // This intentionally nonexistent file also makes feature eligibility explicit. + results := func(status string) string { + return `{"testResults":[{"name":"/outside/missing.test.js","assertionResults":[{"fullName":"works","status":"` + status + `"}]}]}` + } + executor := &jsonExecutor{results: []string{results("passed"), results("failed"), results("failed"), results("failed")}, exits: make([]error, 4)} + run.executor = executor + run.startIntake = func(string, intake.Scenario) (localIntake, error) { + facts := intake.Facts{} + if executor.calls%2 == 1 { + facts = testRun("passed", 0).Facts + } + return &fakeIntake{url: "http://127.0.0.1:1234", findings: facts}, nil + } + var output bytes.Buffer + require.ErrorContains(t, run.Run(t.Context(), &output), "validation is incomplete") + require.Equal(t, 4, executor.calls) + require.Equal(t, "false", executor.envs[0]["DD_CIVISIBILITY_ENABLED"]) + require.Equal(t, "true", executor.envs[1]["DD_CIVISIBILITY_ENABLED"]) + require.Contains(t, output.String(), "Compatibility: inconclusive") + require.Contains(t, output.String(), "Outcomes changed between repeated runs") + paths, err := filepath.Glob(filepath.Join(run.repositoryRoot, ".testoptimization", "testdrive.json")) + require.NoError(t, err) + require.Len(t, paths, 1) + data, err := os.ReadFile(paths[0]) + require.NoError(t, err) + require.True(t, json.Valid(data)) + require.NotContains(t, string(data), "detailed test output") + var result validationResult + require.NoError(t, json.Unmarshal(data, &result)) + require.False(t, result.Success) + require.Len(t, result.Runs, 4) + for i, summary := range result.Runs { + require.Equal(t, executor.commands[i], summary.Command) + require.Equal(t, i%2 == 1, summary.Instrumented) + require.Equal(t, 0, *summary.ExitCode) + } + require.NoDirExists(t, run.platform.(*fakeTracer).sessionDirectory) + html, err := filepath.Glob(filepath.Join(filepath.Dir(paths[0]), "*.html")) + require.NoError(t, err) + require.Equal(t, []string{htmlReportPath(run.repositoryRoot)}, html) + page, err := os.ReadFile(html[0]) + require.NoError(t, err) + require.Contains(t, string(page), "detailed test output") + require.Contains(t, output.String(), "Open report:") + require.NotContains(t, output.String(), "detailed test output") +} + +func TestFinishValidationDoesNotUseSuiteExitAsVerdict(t *testing.T) { + var output bytes.Buffer + result := validationResult{Compatibility: verdict{Status: "compatible"}, Runs: []runSummary{(validationRun{Command: "npm test", ExitCode: 1}).summary()}} + require.NoError(t, finishValidation(&output, t.TempDir(), result)) + require.Contains(t, output.String(), "Results JSON:") +} + +func TestJestRunClosesIntakeOnWriteFailure(t *testing.T) { + run := preparedTestdrive(t) + session, err := NewSession() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, session.Close()) }) + server := &fakeIntake{url: "http://127.0.0.1:1234"} + run.startIntake = func(directory string, _ intake.Scenario) (localIntake, error) { + require.NoError(t, os.RemoveAll(directory)) + return server, nil + } + run.executor = &fakeTestdriveExecutor{} + _, err = run.runJest(t.Context(), &bytes.Buffer{}, session, "/trace/ci/init.js", "baseline", false, intake.Scenario{}, "", "") + require.ErrorIs(t, err, os.ErrNotExist) + require.True(t, server.closed) +} + +func TestJestArgsPreserveNpmForwarding(t *testing.T) { + require.Equal(t, []string{"test", "--", "--json"}, appendJestArgs("npm", []string{"test"}, "--json")) + require.Equal(t, 1, strings.Count(strings.Join(appendJestArgs("npm", []string{"test", "--"}, "--json"), " "), "-- ")) +} + +func withoutTelemetry(run validationRun) validationRun { run.Facts = intake.Facts{}; return run } + +func TestBaselineRemovesQuotedTracerPreloadsAndPreservesOtherOptions(t *testing.T) { + got := stripDatadogNodeOptions(`--require "/project space/node_modules/dd-trace/ci/init.js" --import dd-trace/register.js --require "/project space/setup.js" --max-old-space-size=4096`) + require.Equal(t, `--require "/project space/setup.js" --max-old-space-size=4096`, got) +} + +func TestTelemetryRequiresTheCorrectTestFileAndNoRetries(t *testing.T) { + baseline := withoutTelemetry(testRun("passed", 0)) + instrumented := testRun("passed", 0) + instrumented.Facts.Tests[0].SourceFile = "/other/test.js" + require.Equal(t, "inconclusive", compareJest(baseline, instrumented).Status) + instrumented = testRun("passed", 0) + instrumented.Facts.Tests[0].Attempts[0].Retry = true + require.Equal(t, "inconclusive", compareJest(baseline, instrumented).Status) + baseline.Facts = testRun("passed", 0).Facts + require.Equal(t, "inconclusive", compareJest(baseline, testRun("passed", 0)).Status) +} + +func TestJestRejectsMissingStructuredResults(t *testing.T) { + path := filepath.Join(t.TempDir(), "jest.json") + requireWriteFile(t, path, `{}`) + run := validationRun{} + readJestResults(path, &run) + require.NotEmpty(t, run.ResultError) + requireWriteFile(t, path, `{"testResults":[]}`) + run = validationRun{} + readJestResults(path, &run) + require.Empty(t, run.ResultError, "a skipped suite can legitimately produce an empty results array") +} + +func TestProbeThresholdOverridePreservesFullSuiteAndRecordsAdjustment(t *testing.T) { + run := preparedTestdrive(t) + run.command, run.args = "npm", []string{"test", "--", "--coverage", `--coverageThreshold={"global":{"lines":90}}`} + session, err := NewSession() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, session.Close()) }) + run.startIntake = func(string, intake.Scenario) (localIntake, error) { + return &fakeIntake{url: "http://127.0.0.1:1234"}, nil + } + executor := &jsonExecutor{results: []string{`{"testResults":[]}`, `{"testResults":[]}`}, exits: []error{nil, errors.New("coverage failed")}} + run.executor = executor + full, err := run.runJest(t.Context(), &bytes.Buffer{}, session, "/trace/ci/init.js", "baseline", false, intake.Scenario{}, "", "") + require.NoError(t, err) + words, err := shellquote.Split(full.Command) + require.NoError(t, err) + require.Contains(t, words, `--coverageThreshold={"global":{"lines":90}}`) + require.NotContains(t, full.Command, "--coverageThreshold={}") + require.False(t, full.summary().ProbeCoverageThresholdsDisabled) + require.Empty(t, full.summary().Diagnostic) + probe, err := run.runJest(t.Context(), &bytes.Buffer{}, session, "/trace/ci/init.js", "probe", true, intake.Scenario{}, "probe.test.js", "pass") + require.NoError(t, err) + words, err = shellquote.Split(probe.Command) + require.NoError(t, err) + require.Contains(t, words, "--coverageThreshold={}") + require.NotContains(t, words, `--coverageThreshold={"global":{"lines":90}}`) + require.Contains(t, probe.Command, "--coverage") + require.True(t, probe.summary().ProbeCoverageThresholdsDisabled) + require.Equal(t, "detailed test output", probe.summary().Diagnostic) +} + +func TestProbeJestArgsReplaceBothThresholdForms(t *testing.T) { + for _, flag := range []string{"--coverageThreshold", "--coverage-threshold"} { + for _, args := range [][]string{{"test", "--", flag, `{"global":{"lines":90}}`, "--coverage"}, {"test", "--", flag + `={"global":{"lines":90}}`, "--coverage"}} { + require.Equal(t, []string{"test", "--", "--coverage", "--coverageThreshold={}"}, probeJestArgs("npm", args)) + } + } +} + +func TestCommandDiagnosticRetainsBoundedFailureTail(t *testing.T) { + failure := "Jest: Coverage for statements (0%) does not meet global threshold (62%)" + diagnostic := commandDiagnostic([]byte(strings.Repeat("test output\n", 10000) + "\x1b[31m" + failure + "\x1b[0m\n")) + require.LessOrEqual(t, len([]rune(diagnostic)), 1024) + require.True(t, strings.HasPrefix(diagnostic, "[truncated]")) + require.True(t, strings.HasSuffix(diagnostic, failure)) + require.NotContains(t, diagnostic, "\x1b") + require.Equal(t, diagnostic, (validationRun{Diagnostic: diagnostic}).summary().Diagnostic) +} + +func TestRetryFailureExplainsNativeAndTelemetryMismatch(t *testing.T) { + identity := intake.Test{Module: "vitest", Suite: "probe.test.js", Name: probeName} + var events []intake.Event + for i, status := range []string{"fail", "fail", "pass"} { + tags := map[string]string{"test.module": identity.Module, "test.suite": identity.Suite, "test.name": identity.Name, "test.status": status} + if i > 0 { + tags["test.is_retry"], tags["test.retry_reason"] = "true", "auto_test_retry" + } + events = append(events, intake.Event{Type: "test", Tags: tags}) + } + run := validationRun{Tests: []jestTest{{Name: probeName, Status: "passed"}}, Facts: intake.Facts{Events: events, TestEventCount: 3}} + result := evaluateFeature("auto-retries", run, identity) + require.Equal(t, "failed", result.Status) + require.Contains(t, result.Reason, "Retry telemetry count mismatch: expected 1 failed event, received 2.") + require.Contains(t, result.Reason, "1 passed, 2 failed and 2 retries") + require.Contains(t, result.Reason, "runner has 1 passed, 0 failed") + require.Contains(t, result.Reason, "requires exactly one failed telemetry event") + require.Contains(t, result.Reason, "runner may report only the final passing result") + run.Facts.Events = []intake.Event{events[0], events[2]} + run.Facts.TestEventCount = 2 + require.Equal(t, "passed", evaluateFeature("auto-retries", run, identity).Status) +} diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go index d40f4c55..084fa4c5 100644 --- a/internal/testdrive/multiframework_test.go +++ b/internal/testdrive/multiframework_test.go @@ -37,28 +37,27 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { require.Contains(t, preview.String(), displayName(name)) _, err = os.Stat(filepath.Join(root, ".testoptimization")) require.True(t, os.IsNotExist(err), "preview must be read-only") - if name == "cypress" { + if name == "cypress" || name == "jest" || name == "vitest" { return } // Browser wrapper has its own real-run test. run.nodeVersion = func() string { return "v20.0.0" } installer := &fakeTracer{preloadPath: filepath.Join(root, "isolated")} run.platform = installer run.projectTracer = "" - run.projectTracer = "" - run.projectTracer = "" - run.projectTracer = "" executor := &fakeTestdriveExecutor{} run.executor = executor - run.startIntake = func(string) (localIntake, error) { + run.startIntake = func(string, intake.Scenario) (localIntake, error) { return &fakeIntake{url: "http://127.0.0.1:1234", findings: intake.Facts{TestEventCount: 1, TestCount: 1}}, nil } var output bytes.Buffer - require.NoError(t, run.Run(t.Context(), &output)) + require.ErrorContains(t, run.Run(t.Context(), &output), "validation is incomplete") + require.Contains(t, output.String(), "Compatibility: inconclusive") + require.Contains(t, output.String(), "unvalidated") require.Contains(t, output.String(), displayName(name)+": Passed") require.Contains(t, output.String(), "Tests with coverage: 0 / 1") require.Equal(t, "ddtest-testdrive", executor.env["DD_API_KEY"]) if name == "cucumber" { - require.Equal(t, "true", executor.env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"]) + require.Equal(t, "false", executor.env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"], "reporting-only validation disables behavior-changing features") } switch run.language { case "javascript": @@ -71,7 +70,7 @@ func TestPrepareAllSupportedFrameworks(t *testing.T) { require.NotContains(t, executor.env, "BUNDLE_GEMFILE") require.Contains(t, preview.String(), "Bundler updates Gemfile and Gemfile.lock.") require.NotContains(t, preview.String(), "It will not change") - require.Contains(t, output.String(), "datadog-ci · installed in project") + require.Contains(t, output.String(), "datadog-ci@latest · project bundle installation") require.Contains(t, executor.env["RUBYOPT"], "datadog/ci/auto_instrument") require.NotContains(t, executor.env, "NODE_OPTIONS") } diff --git a/internal/testdrive/repeatability.go b/internal/testdrive/repeatability.go new file mode 100644 index 00000000..4f5e615e --- /dev/null +++ b/internal/testdrive/repeatability.go @@ -0,0 +1,92 @@ +package testdrive + +import ( + "fmt" + "maps" + "path/filepath" + "slices" + "strings" +) + +// Retain recurring per-test differences even when other tests change between +// repeats. This describes observations, not their cause; the verdict stays open. +func compareRepeatedJavaScript(baseline, instrumented, baselineAgain, instrumentedAgain validationRun) verdict { + result := compareJest(baseline, instrumented) + if slices.Equal(outcomeKeys(baseline), outcomeKeys(baselineAgain)) && slices.Equal(outcomeKeys(instrumented), outcomeKeys(instrumentedAgain)) { + return result + } + result.Status = "inconclusive" + result.Reason = "Outcomes changed between repeated runs; flakiness or changing setup prevents attributing the difference to instrumentation." + for _, run := range []validationRun{baseline, instrumented, baselineAgain, instrumentedAgain} { + if run.ResultError != "" { + return result + } + } + type identity struct{ file, name string } + outcomes := [4]map[identity][]string{} + identities := map[identity]bool{} + for i, run := range []validationRun{baseline, instrumented, baselineAgain, instrumentedAgain} { + outcomes[i] = map[identity][]string{} + for _, test := range run.Tests { + key := identity{test.File, test.Name} + identities[key] = true + outcomes[i][key] = append(outcomes[i][key], test.Status+"\n"+test.Failure) + } + for _, values := range outcomes[i] { + slices.Sort(values) + } + } + keys := slices.Collect(maps.Keys(identities)) + slices.SortFunc(keys, func(a, b identity) int { + if c := strings.Compare(a.file, b.file); c != 0 { + return c + } + return strings.Compare(a.name, b.name) + }) + var recurring, variable []string + for _, key := range keys { + a, b, c, d := outcomes[0][key], outcomes[1][key], outcomes[2][key], outcomes[3][key] + if slices.Equal(a, b) && slices.Equal(a, c) && slices.Equal(a, d) { + continue + } + file := key.file + if relative, err := filepath.Rel(baseline.root, file); err == nil && filepath.IsLocal(relative) { + file = relative + } + label := file + " › " + key.name + if slices.Equal(a, c) && slices.Equal(b, d) { + recurring = append(recurring, "Recurring difference in both pairs: "+label+"; baseline: "+outcomeDescription(a)+"; instrumented: "+outcomeDescription(b)) + } else { + variable = append(variable, "Varies between repeats: "+label+"; baseline: "+outcomeDescription(a)+" -> "+outcomeDescription(c)+"; instrumented: "+outcomeDescription(b)+" -> "+outcomeDescription(d)) + } + } + // Keep exit codes and suite-level differences when no test identity explains + // the varying run, rather than pretending a process failure is a test failure. + if len(recurring)+len(variable) > 0 { + result.Differences = append(recurring, variable...) + } + result.Reason += fmt.Sprintf(" %d test identities have recurring differences in both pairs; %d vary between repeats. Recurring differences still require investigation; this does not establish their cause.", len(recurring), len(variable)) + return result +} + +func outcomeDescription(values []string) string { + if len(values) == 0 { + return "not reported" + } + counts := map[string]int{} + for _, value := range values { + counts[value]++ + } + var descriptions []string + for _, value := range slices.Sorted(maps.Keys(counts)) { + status, failure, _ := strings.Cut(value, "\n") + if counts[value] > 1 { + status += fmt.Sprintf(" x%d", counts[value]) + } + if failure != "" { + status += " (" + reportText(failure) + ")" + } + descriptions = append(descriptions, status) + } + return reportText(strings.Join(descriptions, ", ")) +} diff --git a/internal/testdrive/repeatability_test.go b/internal/testdrive/repeatability_test.go new file mode 100644 index 00000000..04d4a7cf --- /dev/null +++ b/internal/testdrive/repeatability_test.go @@ -0,0 +1,65 @@ +package testdrive + +import ( + "bytes" + "encoding/json" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRepeatedComparisonRetainsRecurringDifferencesBesideVariableTests(t *testing.T) { + base := validationRun{root: "/repo", Tests: []jestTest{{File: "/repo/test.ts", Name: "memory", Status: "passed"}, {File: "/repo/test.ts", Name: "timing", Status: "passed"}}} + first := validationRun{ExitCode: 1, Tests: []jestTest{{File: "/repo/test.ts", Name: "memory", Status: "failed", Failure: "expected rounds > 1, received 0"}, {File: "/repo/test.ts", Name: "timing", Status: "failed"}}} + second := validationRun{ExitCode: 1, Tests: []jestTest{first.Tests[0], base.Tests[1]}} + got := compareRepeatedJavaScript(base, first, base, second) + require.Equal(t, "inconclusive", got.Status) + require.Contains(t, got.Reason, "1 test identities have recurring differences") + require.Contains(t, got.Reason, "1 vary between repeats") + require.Contains(t, got.Differences[0], "Recurring difference in both pairs: test.ts › memory") + require.Contains(t, got.Differences[0], "received 0") + require.Contains(t, got.Differences[1], "Varies between repeats: test.ts › timing") + root := t.TempDir() + require.Error(t, finishValidation(&bytes.Buffer{}, root, validationResult{Compatibility: got})) + data, err := os.ReadFile(validationPath(root)) + require.NoError(t, err) + var report validationResult + require.NoError(t, json.Unmarshal(data, &report)) + require.Equal(t, got.Differences, report.Compatibility.Differences) + require.Contains(t, strings.Join(report.Summary.Facts, "\n"), "test.ts › memory") +} + +func TestRepeatedComparisonPreservesProcessAndMissingResultFailures(t *testing.T) { + base := withoutTelemetry(testRun("passed", 0)) + first := testRun("failed", 1) + require.Equal(t, "suspected regression", compareRepeatedJavaScript(base, first, base, first).Status) + missing := validationRun{ExitCode: 1, ResultError: "missing native results"} + got := compareRepeatedJavaScript(base, first, base, missing) + require.Equal(t, "inconclusive", got.Status) + require.NotContains(t, got.Reason, "test identities") + repeat := first + repeat.Tests = nil + got = compareRepeatedJavaScript(base, first, base, repeat) + require.Contains(t, strings.Join(got.Differences, "\n"), "not reported") + pass := testRun("passed", 0) + crash := testRun("passed", 1) + got = compareRepeatedJavaScript(base, crash, base, pass) + require.Equal(t, "inconclusive", got.Status) + require.Contains(t, strings.Join(got.Differences, "\n"), "exit=1") +} + +func TestRepeatedComparisonCountsDuplicateNamesAndBoundsReport(t *testing.T) { + base := validationRun{Tests: []jestTest{{File: "test.ts", Name: "same", Status: "passed"}, {File: "test.ts", Name: "same", Status: "passed"}}} + first := validationRun{Tests: []jestTest{{File: "test.ts", Name: "same", Status: "failed", Failure: strings.Repeat("x", 5000)}}} + got := compareRepeatedJavaScript(base, first, base, base) + require.Contains(t, got.Differences[0], "passed x2") + root := t.TempDir() + require.Error(t, finishValidation(&bytes.Buffer{}, root, validationResult{Compatibility: got})) + data, err := os.ReadFile(validationPath(root)) + require.NoError(t, err) + var report validationResult + require.NoError(t, json.Unmarshal(data, &report)) + require.Less(t, len([]rune(report.Compatibility.Differences[0])), 1100) +} diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index bfbe6624..37bb8106 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -1333,7 +1333,7 @@

Run details

{{ else }}

The command produced no output.

{{ end }} - Open full output file + {{ range .Artifacts }}{{ if eq .Href "test-output.txt" }}Open full output file{{ end }}{{ end }}