diff --git a/Makefile b/Makefile index a63e19a2..3a1d9f89 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ .DEFAULT_GOAL := build -.PHONY: clean fmt vet lint build run release +.PHONY: clean fmt vet lint test build install run release VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X github.com/DataDog/ddtest/internal/buildinfo.Version=$(VERSION) +GOBIN ?= $(shell go env GOBIN) +ifeq ($(strip $(GOBIN)),) +GOBIN := $(firstword $(subst :, ,$(shell go env GOPATH)))/bin +endif clean: go clean -i -x fmt: @@ -14,6 +18,9 @@ test: go test ./... build: test lint go build -ldflags="$(LDFLAGS)" -o ddtest main.go +install: + GOBIN="$(GOBIN)" go install -ldflags="$(LDFLAGS)" . + @echo "Installed ddtest to $(GOBIN)/ddtest" run: go run main.go release: diff --git a/README.md b/README.md index 4b193f7a..65b61a15 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,78 @@ Currently supported: - Python with pytest. - JavaScript with Cucumber, Cypress, Jest, Mocha, Playwright, or Vitest. +## Try Test Optimization locally + +From a supported repository with its test dependencies installed: + +```sh +ddtest onboard +ddtest testdrive +``` + +`onboard` prints GitHub Actions setup instructions. `testdrive` previews its +commands, asks for confirmation, and runs your own tests against a local intake. +It needs no Datadog account, API key, or Agent. Non-interactive callers can review +the preview, then use `ddtest testdrive --yes`. + +All nine frameworks listed above are supported. If a repository contains several +runners, select one with `--framework`. Use `--command` to select a custom entry +point or a small representative part of a large suite: + +```sh +ddtest onboard --framework playwright +ddtest testdrive --framework playwright --command 'npm run test:e2e -- --project=chromium' --yes +``` + +The terminal links to a self-contained HTML report, decoded JSON traffic, and +complete test output under `.testoptimization/testdrive//`. Reports +separate instrumentation success from failed tests, and explicitly indicate when +coverage was not reported. Tracer configuration errors are shown separately. +The local intake supports agentless traffic; Agent/EVP routing is not supported. +Receiving events does not verify test skipping, EFD, or Test Management behavior. +Keep this directory out of source control. Each run has its own files and loopback port. + +Testdrive reuses the project's tracer when the platform's tracer check succeeds. +If the check fails, it attempts to install the latest release inside the session. +`--tracer-version` selects a release or Git revision for that fallback installation; +JavaScript and Python installations leave project dependency files unchanged; +Ruby uses `bundle add datadog-ci`, which updates the project Gemfile and lockfile: + +```sh +ddtest testdrive --tracer-version 6.15.0 --yes # JavaScript example +ddtest testdrive --tracer-version 'git:' --yes +``` + +The same option works for JavaScript, Python, and Ruby. Git selections also accept +branches and tags; they require Git and the tracer's source-build prerequisites. + +Local testdrive prerequisites: + +- JavaScript: Node.js 22+, npm if `dd-trace` needs to be installed, + and the project's package manager and dependencies. Vitest/ESM loading and + Cypress config/support wrappers are supplied automatically. +- Python: an activated project environment with pytest, and pip if `ddtrace` is + absent. Fallback installation uses a session-owned directory; the active + environment is unchanged. +- Ruby: Ruby/Bundler. If `datadog-ci` is unavailable, testdrive runs + `bundle add datadog-ci` in the project, honoring `BUNDLE_GEMFILE` and the + existing bundle settings. Bundler updates the project Gemfile and lockfile; + tests use that same bundle. Native gem builds need build tools. +- Browser suites: install the project's browsers and start any required services + first, or use its existing test command that manages them. Testdrive does not + install browsers or start applications on its own. + +Project dependency manifests and lockfiles are not edited by testdrive. Testdrive +uses the framework’s normal command; pass `--command` to run a package script and +its lifecycle hooks or other custom setup. Tracer downloads require network access. This release covers root projects and GitHub Actions onboarding; +monorepo orchestration and other CI providers are outside this scope. + +See the [Milestone 2 validation record](docs/testing/onboarding-milestone-2.md) +for tested repository commits, commands, and limitations. + ## Prerequisites -Before using DDTest, you must have **Datadog Test Optimization** already set up and enabled with a Datadog Test Optimization library for your language and framework. DDTest relies on this integration to discover your tests and plan test execution accordingly. +Before using `ddtest plan` or `ddtest run`, you must have **Datadog Test Optimization** already set up and enabled with a Datadog Test Optimization library for your language and framework. DDTest relies on this integration to discover your tests and plan test execution accordingly. Minimum supported library and runtime requirements: diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md new file mode 100644 index 00000000..9e013d7d --- /dev/null +++ b/docs/design/agent-driven-onboarding.md @@ -0,0 +1,230 @@ +# Agent-driven onboarding + +Status: Milestones 0, 1, and 2 implemented; Milestones 3 and 4 proposed + +Last updated: 2026-09-22 + +## Goal + +A user should be able to give a coding agent one prompt: + +> Onboard test optimization using ddtest. + +The agent discovers the flow through `ddtest help`, makes the small CI edit suggested by `ddtest onboard`, runs `ddtest testdrive`, and gives the user a clickable report showing their own tests. + +The local testdrive requires no Datadog account, API key, or Agent. Credentials come last, are configured by a human in CI, and are never shown to the coding agent. + +## Product rules + +- Use the customer's real test suite, not a demo. +- Keep the main flow to `ddtest onboard` and `ddtest testdrive`. +- Detect the repository instead of asking setup questions. +- Show commands and filesystem changes before running them. +- Keep JavaScript and Python tracer installations isolated; disclose that Ruby fallback uses bundle add and updates project dependency files. +- Explain what worked, what did not, and the next useful action. +- Give the user clickable local and CI links. +- Dogfood a thin end-to-end slice before adding abstractions. + +For now, do not build a generic onboarding framework, YAML editor, public result schema, compatibility matrix, or exhaustive error taxonomy. + +## What exists today + +The working preview is in draft PR #128 on `anmarchenko/agentic-onboarding-runbook`. + +### User flow + +- `ddtest help` points to `ddtest onboard`. +- `ddtest onboard` detects all nine supported platform/framework pairs and candidate GitHub Actions test workflows; the coding agent inspects the actual jobs. Repositories with several frameworks use `--framework`. +- It prints repository-owned Markdown instructions containing the concrete workflow edit, asks the agent to run `ddtest testdrive`, and tells the agent to post every report link to the user. +- The local testdrive comes before the human API-key and GitHub-secret steps. +- `ddtest testdrive` previews its commands and file changes. Interactive terminals ask once for confirmation. Non-interactive callers must review the preview and rerun with `--yes`. +- `make install` builds and installs DDTest into the current user's Go bin directory on macOS and Linux. + +### Local testdrive + +- Each run has a unique directory and kernel-assigned loopback port, so sessions can run concurrently. +- Every language reuses the project tracer when its standard platform check succeeds. If it fails, install latest or a release/Git revision selected with `--tracer-version`. JavaScript and Python install inside the session; Python keeps the selected interpreter. Ruby runs `bundle add datadog-ci` and tests against the project bundle, updating its Gemfile and lockfile. +- The local intake supports the endpoints exercised by that tracer and enables Test Optimization, coverage, Intelligent Test Runner, Early Flake Detection, Auto Test Retries, Impacted Tests, failed-test replay, and Test Management. +- Test events and test- or suite-level coverage are decoded. Raw multipart or msgpack payloads are not retained; saved traffic is JSON only. +- Complete test output is saved separately. + +The terminal and self-contained HTML report answer: + +- Did any tests fail? +- Are any tests flaky because they passed on retry? +- Are any tests unusually slow compared with the median? +- Do any tests or suites cover unusually many files compared with the median? + +Problem cards appear only when the answer is yes. Affected tests are always visible and expand to show attempts, timings, errors, retry information, source excerpts, and coverage. Flaky tests use `test.final_status` and are not also reported as failed. Covered files appear one per line and are paginated 50 at a time. Separate paginated tabs list all suites and tests. + +The report also shows event and coverage counts, framework result, project-tracer reuse or the fallback tracer selection, saved JSON traffic, and test output. `testdrive` prints it as an absolute clickable `file://` link. + +### Code map + +- `internal/onboard/`: repository detection and embedded Markdown instructions. +- `internal/testdrive/`: session lifecycle, preview, execution, terminal output, and HTML report. +- `internal/testdrive/intake/`: local intake, JSON capture, event and coverage decoding, and findings. +- `internal/platform/` and `internal/framework/`: platform-owned tracer detection and installation, platform/framework detection, and test commands. + +### Evidence and limits + +- Existing integration tests run the public CLI with real tracers against fixtures for all nine frameworks. Coverage is reported only when supplied by the tracer. +- A concurrent integration test proves port, traffic, and file isolation. +- Unit tests cover decoding, final status, flaky tests, both coverage granularities, medians, source excerpts, HTML rendering, and confirmation behavior. +- Dogfooding on React Native Paper recognized 1,363 events and coverage for all 680 logical tests, including Early Flake Detection retries. + +Public support covers all nine pairs in Milestone 2, with GitHub Actions onboarding. The intake is not a complete Datadog backend. There is no upload service, Datadog forwarding, stable JSON contract, or savings calculation. Onboarding instructions are copied from the onboarding MCP source rather than shared with it. + +Every change must pass: + +```shell +make test +make lint +``` + +## Milestone 0: prove the local loop — complete + +Milestone 0 proved the risky path before designing the product around it: + +- start a minimal local Test Optimization intake on port `0`; +- install and run a real pinned JavaScript tracer; +- receive a real Jest test event and coverage; +- save each run in its own session directory; +- run two sessions concurrently without collisions; +- require no Datadog credentials or Agent. + +## Milestone 1: delightful Jest preview — implemented + +Milestone 1 turned the spike into the current `onboard` and `testdrive` flow described above. + +Its important interaction contract is: + +- detection and preview happen before any write or external command; +- `ddtest testdrive --yes` is the explicit non-interactive path; +- running the command is one decision—there is no persisted plan, checksum, approval file, or second execution command; +- instrumentation success is independent of whether customer tests pass; +- the project manifest and lockfile are never changed; +- the agent posts the report link to the user. + +Keep dogfooding Jest while later milestones are built. Fix repeated real problems directly; extract shared types only when another working implementation needs them. + +## Milestone 2: every supported platform/framework pair — implemented + +Extend the same basic onboarding and testdrive to the pairs already supported by DDTest: + +| Platform | Frameworks | +| --- | --- | +| JavaScript | Jest, Mocha, Cypress, Playwright, Cucumber, Vitest | +| Python | pytest | +| Ruby | RSpec, Minitest | + +Implemented in vertical slices: + +1. Reuse each platform and framework's `Detect` method and test command. Remove Jest-specific names from the shared report. +2. Add the remaining JavaScript frameworks using the existing isolated `dd-trace` installation. +3. Add one pinned isolated `ddtrace` installation for pytest. +4. Use bundle add for Ruby tracer installation shared by RSpec and Minitest. +5. Add a tiny real-tracer fixture for every pair and dogfood at least one real repository per platform. + +For every pair, `onboard` finds the relevant GitHub Actions job and prints one small setup. `testdrive` previews and runs the detected framework's normal command (or the explicit `--command` entry point), treats received events as proof even when tests fail, reports missing coverage honestly, and preserves the same terminal and HTML experience where the tracer supplies the data. + +Do not solve monorepos, new CI providers, or cross-platform tracer abstractions here. One known-good tracer version per platform is enough. + +All nine pairs have completed the credential-free flow with JSON traffic and local reports. Manifests and lockfiles are checked for preservation. The validation record distinguishes full-suite runs from representative browser/RSpec subsets and records the pinned Cucumber tracer workaround. + +## Milestone 3: guided Test Parallelization onboarding + +Starting prompt: + +> Onboard test parallelization using ddtest. + +Add `ddtest onboard parallelization` as the obvious next step after Test Optimization works. If Test Optimization is not configured, point back to `ddtest onboard`; do not combine both migrations. + +The command detects the platform, framework, and GitHub Actions test job, then tells the coding agent how to make one concrete transformation using the existing product: + +1. A plan job installs the project as CI already does and runs `ddtest plan`. +2. The plan job exposes DDTest's generated matrix and uploads `.testoptimization/`. +3. A matrix job downloads the artifact and runs `ddtest run --ci-node ${{ matrix.ci_node_index }}`. +4. The old command is removed so CI does not run the full suite twice. + +Start with one worker per CI node, `fail-fast: false`, explicit minimum and maximum parallelism, and the existing CI-job overhead model. Preserve runtime setup, environment, services, caches, permissions, timeouts, and artifacts. DDTest prints instructions; the agent edits the workflow. Do not build a YAML rewriting engine or another planner. + +The first real GitHub Actions run is the test drive. The agent gives the user its link and reports: + +- whether the plan job and every node passed; +- selected node count; +- estimated full-suite and parallel wall time; +- modeled CI overhead and imbalance; +- dedicated slow-suite runners, if any; +- the smallest corrective edit for a concrete setup failure. + +Selecting one node is a valid success when extra nodes would not help enough. Support the same nine pairs as Milestone 2. Stop clearly when an existing matrix or parallel runner cannot be combined safely. + +Milestone 3 ships when an agent can discover the flow from the starting prompt, make a reviewable GitHub Actions edit, run the existing planner and runner, and return a clickable CI link with the important plan facts. + +## Milestone 4: `dd-trace-js` runbook parity + +Match the useful conclusions of the validation runbook, not its internal architecture. Keep `ddtest onboard` and `ddtest testdrive`; do not copy its manifest, execution-plan, checksum, approval-file, persisted-lock, or exit-code machinery. + +The terminal and local report show five independent conclusions: + +- **Basic Reporting:** the tracer reports a real project test. +- **CI configuration:** the selected job visibly initializes Test Optimization and configures transport. +- **Early Flake Detection:** a new passing test is retried with the expected reason. +- **Auto Test Retries:** a fail-once test passes on retry with the expected reason. +- **Test Management:** a configured test is matched and tagged as quarantined. + +Each conclusion is simply works, needs attention, or could not be checked. Name the exact missing prerequisite, first useful action, and cleanup status. Keep this validation scope separate from code-coverage counts. + +Implementation order: + +1. Add all five conclusions to the simplest Jest repository and dogfood the complete flow before generalizing it. +2. Keep the normal happy path: the customer's instrumented suite proves Basic Reporting. +3. After Basic Reporting succeeds, run small DDTest-owned tests for the three advanced features. The local intake supplies settings, known tests, and managed tests; emitted attempts and events must prove behavior. +4. If Basic Reporting is inconclusive, compare one representative test without and with instrumentation and debug logging. Do not double every successful testdrive. +5. Audit the selected GitHub Actions job without executing it. Resolve the setup DDTest generates, direct commands, and simple local package scripts; report dynamic or remote wrappers as inconclusive. +6. Extend the proven Jest slice to Mocha, Cypress, Playwright, Cucumber, and Vitest. + +Temporary tests appear in the preview, are created only after confirmation, and are removed after the run. Their decoded JSON events and output remain in the session. Browser- or application-backed checks may be inconclusive with the missing prerequisite named; DDTest does not start applications or install browsers implicitly. + +Python and Ruby retain Milestone 2's Basic Reporting and suite analysis until their tracer behavior and real dogfood cases justify equivalent advanced checks. + +Milestone 4 ships when all six JavaScript frameworks report the five conclusions without Datadog credentials, preserve project dependencies and concurrent-session isolation, clean up temporary tests, and provide clickable local and CI links. + +## Next work + +Milestone 2 now has public-CLI real-tracer fixtures for every pair and independent open-source dogfood runs. See [the validation record](../testing/onboarding-milestone-2.md). The next planned feature is Milestone 3; keep the existing nine-pair flow green. + +For each slice: + +1. add one real-tracer fixture; +2. run it in a real repository; +3. record what the human or agent had to guess; +4. fix observed friction; +5. run `make test`, `make lint`, and the installed binary. + +## Later ideas + +These should not delay Milestones 2–4: + +- Homebrew distribution. +- One offline source for the Markdown instructions currently duplicated with the onboarding MCP implementation in `dd-source`. +- A broad `ddeval` suite built from real repository and CI shapes. +- Fully local TIA backed by SQLite coverage history, considering committed, staged, unstaged, and untracked changes. +- Real Datadog mode when `DD_API_KEY` is present. +- Local reproduction of CI operating-system and runtime tags. +- Historical test analysis through the Datadog API. +- `ddtest doctor` as a reusable diagnostic command if the integrated flow proves it is needed. +- More CI providers and monorepo orchestration. +- Multiple tracer-version support. +- Local savings estimates and historical replay. +- A hosted Testdog page for sharing a report without a Datadog account. + +Choose the next slice from what users struggle with after Milestone 4, not from a speculative architecture. + +## References + +- `~/p/shepherd/tools/mockdog`: local Test Optimization intake precedent. +- `~/p/dd-trace-js/ci/runbook.md`: validation conclusions and JavaScript adapter behavior. +- `~/p/test-visibility-install-script`: isolated JavaScript tracer installation precedent. +- The onboarding MCP instructions in `~/dd/dd-source`: current source material for CI setup instructions. diff --git a/docs/testing/onboarding-milestone-2.md b/docs/testing/onboarding-milestone-2.md new file mode 100644 index 00000000..2edfc873 --- /dev/null +++ b/docs/testing/onboarding-milestone-2.md @@ -0,0 +1,201 @@ +# Milestone 2 validation — 2026-09-22 + +All nine supported frameworks completed the public `onboard → testdrive --yes → +report.html` flow on independent open-source repositories. Both commands exited +zero for every row below. Each run received real tracer events, retained decoded +JSON traffic and test output, and generated a local HTML report. + +This was a manually selected open-source sample, not a statistically random +compatibility study. Browser and RSpec checks used representative subsets as +identified below. These results do not imply every version or configuration of +each framework is compatible. + +This historical record predates project-tracer reuse. Current testdrive always uses +a project tracer when present and installs latest (or `--tracer-version`) only as a +fallback. The pinned versions below describe the recorded runs. + +## Environment and checks + +- macOS arm64; Go 1.26.5; Node.js 24.14.0; Python 3.11; Ruby 3.4.7. +- Pinned tracers: `dd-trace@6.15.0`, `ddtrace==4.15.1`, `datadog-ci@1.39.0`. +- `make test` and `make lint` pass. +- The opt-in public CLI matrix runs real fixtures for all nine frameworks, + including paths containing spaces for JavaScript/Python, existing Ruby + lockfiles, a Cucumber Background, and preserved Cypress task/after-run hooks. +- Existing instrumented Jest and concurrent-session isolation checks pass. +- Root manifests and lockfiles were hashed immediately before and after each + open-source run, including presence/absence. All nine snapshots were unchanged in those recorded runs. Ruby now uses + `bundle add` for fallback installation, which intentionally updates its Gemfile + and lockfile; the earlier Ruby isolation result no longer describes this behavior. + Dependency installation and application builds happened before those snapshots. +- No Datadog credentials or Agent were used. This validates local instrumentation + and generated onboarding guidance, not a connection to a live Datadog account. + +## Open-source results + +“Covered” means a test had associated coverage in received traffic. Zero means +coverage was not reported, not that the test covered no application code. Events +can exceed logical tests because the local intake enables retries. + +| Framework | Repository / exact commit | Scope | Logical tests | Events | Covered | +| --- | --- | --- | ---: | ---: | ---: | +| jest | [typestack/class-validator](https://github.com/typestack/class-validator/tree/2e1a5c27dbd65b80e27fe96b49bd6e6641fa3603) | Full Jest suite | 743 | 1549 | 743 | +| mocha | [omichelsen/compare-versions](https://github.com/omichelsen/compare-versions/tree/98e81116ef4197b42dca8c3fde8d1e8166e2a81f) | Full Mocha suite | 336 | 678 | 0 | +| vitest | [unjs/destr](https://github.com/unjs/destr/tree/541b6f9aeada9fc30de9c5a7e086dbfc1c6fcdc7) | Full Vitest suite (lint omitted) | 22 | 44 | 22 | +| playwright | [dropbox/ttvc](https://github.com/dropbox/ttvc/tree/239481cdecb841f63f0b3f3e13be52275333b18d) | One Chromium browser test | 1 | 2 | 0 | +| cypress | [cypress-io/cypress-example-kitchensink](https://github.com/cypress-io/cypress-example-kitchensink/tree/ddaaa92080b68d71d7a1797b4ed20ada18ed2a2a) | Six Todo browser tests | 6 | 12 | 0 | +| cucumber | [cloudevents/sdk-javascript](https://github.com/cloudevents/sdk-javascript/tree/bf5d53f2862248d72d9869cf5173f51b32583faf) | HTTP/Kafka conformance suite | 7 | 14 | 0 | +| pytest | [pallets/itsdangerous](https://github.com/pallets/itsdangerous/tree/672971d66a2ef9f85151e53283113f33d642dabd) | Full pytest suite | 297 | 297 | 297 | +| rspec | [ruby-concurrency/concurrent-ruby](https://github.com/ruby-concurrency/concurrent-ruby/tree/e674fb2688206bb7cb66dee108e1d08184413afb) | AtomicBoolean spec file | 25 | 25 | 25 | +| minitest | [ruby-i18n/i18n](https://github.com/ruby-i18n/i18n/tree/547917dd8d41fab781a81880f22687fc4eac5d85) | Full Minitest suite | 1607 | 1607 | 1607 | + +## Reproduction + +Build DDTest with Go 1.26.5 and put the resulting binary on PATH. Check out each +linked commit and install that project's dependencies first. Use Node.js 22+ for +the JavaScript tracer and activate the project's Python environment for pytest. +The Ruby checks use Ruby 3.4.7 with native extension build tools. + +Repository preparation used: + +- class-validator and compare-versions: `npm ci --no-audit --no-fund`. +- destr: `pnpm install --frozen-lockfile` (pnpm 10); the run selects Vitest directly. +- ttvc: `npm install --no-audit --no-fund`, `npm run build`, and + `node_modules/.bin/playwright install chromium`. The existing Playwright config + starts its application through `yarn express`. The npm preparation updated its + Yarn lockfile; the testdrive itself left the prepared dependency files unchanged. +- Cypress Kitchen Sink: `npm ci --no-audit --no-fund`, then `npm start` on port 8080. + Browser installation is part of project preparation, not a testdrive operation. +- CloudEvents: `git submodule update --init --depth 1`, `npm ci --no-audit --no-fund`, + `npm run build:schema`, then `npm run build:src`. Its conformance submodule was + `eddc279339609ed92d128bcd2b0d5c558a7ce396`. +- itsdangerous: create/activate a Python 3.11 venv and run + `python -m pip install -e . pytest freezegun`. +- concurrent-ruby and i18n: those recorded runs used the former isolated bundle. + Current Ruby testdrives use the project bundle and add a missing tracer with + `bundle add datadog-ci`. + +Run `ddtest onboard --framework FRAMEWORK` in each repository, followed by the +commands below. Package scripts used in the recorded runs are selected explicitly +with `--command`; automatic detection does not override framework commands. + +**typestack/class-validator** + +```sh +ddtest testdrive --framework jest --yes --command 'npm test -- --runInBand' +``` + +**omichelsen/compare-versions** + +```sh +ddtest testdrive --framework mocha --yes --command 'npm test' +``` + +**unjs/destr** + +```sh +ddtest testdrive --framework vitest --yes --command 'node_modules/.bin/vitest run' +``` + +**dropbox/ttvc** + +```sh +ddtest testdrive --framework playwright --yes --command 'node_modules/.bin/playwright test test/e2e/stylesheet1 --project=chromium --workers=1' +``` + +**cypress-io/cypress-example-kitchensink** + +```sh +ddtest testdrive --framework cypress --yes --command 'node_modules/.bin/cypress run --spec cypress/e2e/1-getting-started/todo.cy.js' +``` + +**cloudevents/sdk-javascript** + +```sh +ddtest testdrive --framework cucumber --yes --command 'npm run conformance' +``` + +**pallets/itsdangerous** + +```sh +ddtest testdrive --framework pytest --yes +``` + +**ruby-concurrency/concurrent-ruby** + +```sh +ddtest testdrive --framework rspec --yes --command 'bundle exec rspec spec/concurrent/atomic/atomic_boolean_spec.rb' +``` + +**ruby-i18n/i18n** + +```sh +ddtest testdrive --framework minitest --yes +``` + +Each run prints an absolute `Open report:` link. The report, `test-output.txt`, +and decoded `intake/*.json` files remain under its unique +`.testoptimization/testdrive//` directory. + +Run the reproducible fixture matrix from DDTest itself: + +```sh +GOTOOLCHAIN=go1.26.5 make test +GOTOOLCHAIN=go1.26.5 make lint +GOTOOLCHAIN=go1.26.5 \ + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1 \ + DDTEST_RUN_NPM_INTEGRATION_TEST=1 \ + go test ./internal/testdrive \ + -run 'TestPublicFrameworkTestdrives|TestInstrumentedJestFixture' \ + -count=1 -v +``` + +The integration matrix installs project dependencies in temporary directories, +including Cypress's browser. Python, pip/venv, Ruby, Bundler, Node, npm, and native +gem build tools must already be installed. + +## Problems found and addressed + +- Python's `uv run tox` and Ruby's `bundle exec rake` workflows were initially + missed. They are now included as candidate workflows for the coding agent to + inspect; this does not claim to statically validate every CI job. +- The pinned JavaScript tracer dereferenced a missing `scenario.id` for Cucumber + Background nodes when impacted-test detection was enabled. Local Cucumber runs + disable that feature, and generated CI guidance carries the same workaround. + CloudEvents' conformance scenarios and the independent + [Cucumber 7 TypeScript starter](https://github.com/hdorgeval/cucumber7-ts-starter/tree/02fb70d50ce0ccbad7ccc51fc856362f5efc5470) + passed after this change. The starter has no GitHub Actions workflow, so its + extra check covered testdrive only. A Background is now in the fixture matrix. +- Returning a completely empty known-tests map disabled Jest's existing retry + behavior. The intake now supplies empty datasets for all supported runner names; + the existing Jest retry and concurrent-session assertions remain intact. +- Cypress starts its config process beside the generated wrapper. The wrapper now + restores the project working directory before loading the original config, so + relative file access and existing after-run hooks keep their original behavior. + The fixture asserts both a customer task and an after-run file write. +- One Ruby fixture failed in a temporary directory named `project space`, not + in the DDTest checkout. The captured run used Ruby 3.4.7, Bundler 4.0.16, + `datadog-ci` 1.39.0, `datadog` 2.42.0, and `libdatadog` 40.0.0.2.0 on arm64 macOS. + The failing extension was `datadog-2.42.0/ext/libdatadog_api`. After + `ruby extconf.rb`, RubyGems ran + `make DESTDIR= sitearchdir=./.gem.20260922-66018-82y98z sitelibdir=./.gem.20260922-66018-82y98z`. + While compiling `crashtracker.c`, Clang reported `no such file or directory` + for an include path beginning `space/.testoptimization/testdrive/` and ending + `lib/pkgconfig/../../include`; make exited with `crashtracker.o Error 1`. + This suggests an incorrectly escaped build path. The log does not contain the + full compiler invocation, so the responsible escaping step was not established. + It does not establish a general Ruby path limitation or explain failures in + checkouts without spaces. Testdrive now attempts the build and preserves + Bundler's actual error instead of rejecting paths in advance. +- Ruby fallback now invokes `bundle add datadog-ci` using the project's Bundler + settings. Bundler owns Gemfile/lockfile changes, dependency resolution and gem + installation; DDTest does not construct or copy a separate bundle. + +## Deliberate limits + +Root projects and GitHub Actions onboarding only. No monorepo orchestration, +implicit application/browser installation, live Datadog connection validation, +public result schema, savings estimates, or Milestone 3/4 feature guarantees. +Only macOS arm64 was used for this open-source sample. Coverage depends on the +framework/tracer and the project's existing coverage setup; unsupported or absent +coverage is shown explicitly rather than inferred from passing tests. diff --git a/go.mod b/go.mod index da7a067d..606ac794 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/tinylib/msgp v1.6.5 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 + go.yaml.in/yaml/v3 v3.0.5 ) require ( @@ -27,6 +28,5 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/text v0.42.0 // indirect ) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 2af5b770..6f5a4c6a 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -31,9 +31,17 @@ import ( var defaultParallelism = settings.DefaultParallelism() var rootCmd = &cobra.Command{ - Use: "ddtest", - Short: "A test runner from Datadog", - Long: "Command line tool for running tests with Datadog Test Optimization.", + Use: "ddtest", + Short: "A test runner from Datadog", + Long: `Command line tool for running tests with Datadog Test Optimization. + +Start here: + + cd + ddtest onboard + +The onboard command detects the test and CI setup, explains the required +changes, points to local verification, and tells you when human action is needed.`, Version: buildinfo.CurrentVersion(), PersistentPreRunE: runPersistentPreRun, } @@ -154,6 +162,7 @@ func init() { rootCmd.AddCommand(planCmd) rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(onboardCmd) rootCmd.AddCommand(testdriveCmd) cobra.OnInitialize(settings.Init) diff --git a/internal/cmd/cmd_test.go b/internal/cmd/cmd_test.go index 3046ae8d..ce1383f7 100644 --- a/internal/cmd/cmd_test.go +++ b/internal/cmd/cmd_test.go @@ -274,23 +274,17 @@ func TestRootCommandFlags(t *testing.T) { } func TestCommandHierarchy(t *testing.T) { - // Verify that planCmd and runCmd are added to rootCmd + // Verify that the public commands are added to rootCmd. commands := rootCmd.Commands() - var foundPlan, foundRun bool + found := make(map[string]bool) for _, cmd := range commands { - if cmd.Name() == "plan" { - foundPlan = true - } - if cmd.Name() == "run" { - foundRun = true - } + found[cmd.Name()] = true } - if !foundPlan { - t.Error("plan command should be added to root command") - } - if !foundRun { - t.Error("run command should be added to root command") + for _, name := range []string{"onboard", "plan", "run", "testdrive"} { + if !found[name] { + t.Errorf("%s command should be added to root command", name) + } } } @@ -560,8 +554,15 @@ func TestExecute(t *testing.T) { } output := buf.String() - if !strings.Contains(output, "ddtest") { - t.Error("help output should contain command name 'ddtest'") + for _, expected := range []string{ + "Start here:", + "cd ", + "ddtest onboard", + "onboard Start here: onboard this repository to Test Optimization", + } { + if !strings.Contains(output, expected) { + t.Errorf("help output should contain %q:\n%s", expected, output) + } } } @@ -763,8 +764,8 @@ func TestCommandUsage(t *testing.T) { } // Expected commands (cobra adds completion and help automatically) - expectedCommands := []string{"plan", "run", "testdrive"} - requiredCommands := []string{"completion", "help [command]", "plan", "run", "testdrive"} + expectedCommands := []string{"onboard", "plan", "run", "testdrive"} + requiredCommands := []string{"completion", "help [command]", "onboard", "plan", "run", "testdrive"} // Verify minimum expected commands exist for _, expected := range expectedCommands { diff --git a/internal/cmd/onboard.go b/internal/cmd/onboard.go new file mode 100644 index 00000000..dd77d4a3 --- /dev/null +++ b/internal/cmd/onboard.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "io" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/spf13/cobra" +) + +var onboardCmd = newOnboardCommand(func(output io.Writer) error { + return onboard.Run(output) +}) + +func newOnboardCommand(run func(io.Writer) error) *cobra.Command { + return &cobra.Command{ + Use: "onboard", + Short: "Start here: onboard this repository to Test Optimization", + Long: "Detects the supported test setup and prints the smallest Datadog Test Optimization onboarding instructions. It does not edit files.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return run(cmd.OutOrStdout()) + }, + } +} diff --git a/internal/cmd/onboard_test.go b/internal/cmd/onboard_test.go new file mode 100644 index 00000000..5de56b54 --- /dev/null +++ b/internal/cmd/onboard_test.go @@ -0,0 +1,36 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "bytes" + "io" + "testing" +) + +func TestOnboardCommandRunsDetection(t *testing.T) { + called := false + command := newOnboardCommand(func(output io.Writer) error { + called = true + _, _ = io.WriteString(output, "onboarding instructions\n") + return nil + }) + if command.Short != "Start here: onboard this repository to Test Optimization" { + t.Fatalf("Short = %q", command.Short) + } + var output bytes.Buffer + command.SetOut(&output) + + if err := command.ExecuteContext(t.Context()); err != nil { + t.Fatalf("ExecuteContext() unexpected error: %v", err) + } + if !called { + t.Fatal("onboard command did not run detection") + } + if output.String() != "onboarding instructions\n" { + t.Fatalf("output = %q", output.String()) + } +} diff --git a/internal/onboard/instructions/github.md b/internal/onboard/instructions/github.md new file mode 100644 index 00000000..be45402b --- /dev/null +++ b/internal/onboard/instructions/github.md @@ -0,0 +1,43 @@ +# Enable Datadog Test Optimization for __FRAMEWORK__ on GitHub Actions + +Apply this setup to every GitHub Actions job that runs __FRAMEWORK__. + +## 1. Instrument the test job + +Add this step after checkout and dependency installation, immediately before the first test step: + +```yaml +- name: Configure Datadog Test Optimization + uses: datadog/test-visibility-github-action@v3 + with: + languages: __LANGUAGE__ + __TRACER_SETTING__ + api_key: ${{ secrets.DD_API_KEY }} + site: datadoghq.com +``` + +If the organization uses a Datadog site other than US1, replace `datadoghq.com` with that site. + +__BOOTSTRAP__ + +Keep the existing test command and unrelated workflow content unchanged. Add the Datadog action once per test job, not once per test step. + +## 2. Try it locally + +Run the local, credential-free setup check: + +```shell +ddtest testdrive +``` + +After it finishes, post every `Open report:` link printed by `ddtest testdrive` to the user so they can open the local Test Optimization report. Do not only summarize the result. + +## 3. Ask a human to connect Datadog + +The API key must be created and added to GitHub by a human. Ask the human to: + +1. Follow the site-neutral [Datadog API key instructions](https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token) and create the key in the selected Datadog site. +2. Add it to the GitHub repository as a secret named `DD_API_KEY`. +3. Tell you when the secret is ready without sharing the key itself. + +Do not ask the human to paste the API key into chat, and do not try to create or read the secret yourself. After the human confirms it is ready, commit and push the workflow change. The GitHub Actions run verifies the real Datadog backend connection. diff --git a/internal/onboard/multiframework_test.go b/internal/onboard/multiframework_test.go new file mode 100644 index 00000000..fa96d902 --- /dev/null +++ b/internal/onboard/multiframework_test.go @@ -0,0 +1,57 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOnboardAllSupportedFrameworks(t *testing.T) { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest", "rspec", "minitest"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + language := "js" + entry := name + files := map[string]string{"package.json": `{"scripts":{"test":"` + name + `"}}`} + switch name { + case "pytest": + files = map[string]string{"pyproject.toml": "[tool.pytest.ini_options]\n"} + language = "python" + entry = "uv run tox" + case "rspec", "minitest": + files = map[string]string{"Gemfile": "gem '" + name + "'\n"} + language = "ruby" + entry = "bundle exec rake" + } + for file, contents := range files { + require.NoError(t, os.WriteFile(filepath.Join(root, file), []byte(contents), 0644)) + } + require.NoError(t, os.MkdirAll(filepath.Join(root, ".github", "workflows"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".github", "workflows", "ci.yml"), []byte("jobs:\n tests:\n steps:\n - run: "+entry+"\n"), 0644)) + var output bytes.Buffer + t.Chdir(root) + require.NoError(t, Run(&output)) + require.Contains(t, output.String(), "languages: "+language) + require.Contains(t, output.String(), language+"-tracer-version: '' # Latest release") + require.Contains(t, output.String(), "ddtest testdrive --framework "+name) + require.Contains(t, output.String(), "Ask a human to connect Datadog") + if language != "js" { + require.NotContains(t, output.String(), "NODE_OPTIONS") + } + if name == "vitest" { + require.Contains(t, output.String(), "DD_TRACE_ESM_IMPORT") + } + if name == "cypress" { + require.Contains(t, output.String(), "setupNodeEvents") + } + }) + } +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go new file mode 100644 index 00000000..1bd97176 --- /dev/null +++ b/internal/onboard/onboard.go @@ -0,0 +1,221 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +// Package onboard prints the smallest supported Test Optimization setup. +package onboard + +import ( + _ "embed" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/DataDog/ddtest/internal/platform" + "go.yaml.in/yaml/v3" +) + +const githubAction = "datadog/test-visibility-github-action" + +//go:embed instructions/github.md +var gitHubInstructions string + +// Run detects the first supported onboarding path and prints its instructions. +func Run(output io.Writer) error { + repositoryRoot, err := os.Getwd() + if err != nil { + return fmt.Errorf("find repository root: %w", err) + } + detectedPlatform, err := platform.DetectPlatform() + if err != nil { + return err + } + runner, err := detectedPlatform.DetectFramework() + if err != nil { + return err + } + language := detectedPlatform.Name() + name := runner.Name() + + workflows, configured, err := findWorkflows(repositoryRoot, language, name) + if err != nil { + return err + } + if len(workflows) == 0 { + return fmt.Errorf("onboard could not find a GitHub Actions workflow that runs %s", name) + } + + _, _ = fmt.Fprintf(output, "DDTest found %s, %s, and GitHub Actions.\n", language, name) + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, "Test workflow(s):") + for _, workflow := range workflows { + _, _ = fmt.Fprintf(output, " - %s\n", workflow) + } + + if len(configured) == len(workflows) { + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, "Datadog Test Optimization already appears in every detected test workflow.") + _, _ = fmt.Fprintf(output, "Run `ddtest testdrive --framework %s` to check the setup locally.\n", name) + _, _ = fmt.Fprintln(output, "After it finishes, post every `Open report:` link to the user so they can open the local Test Optimization report.") + return nil + } + + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, strings.TrimSpace(instructions(language, name))) + return nil +} + +func findWorkflows(repositoryRoot, language, name string) ([]string, []string, error) { + var workflows []string + var configured []string + directory := filepath.Join(repositoryRoot, ".github", "workflows") + entries, err := os.ReadDir(directory) + if os.IsNotExist(err) { + return workflows, configured, nil + } + if err != nil { + return nil, nil, fmt.Errorf("find GitHub Actions workflows: %w", err) + } + for _, entry := range entries { + if entry.IsDir() || (filepath.Ext(entry.Name()) != ".yml" && filepath.Ext(entry.Name()) != ".yaml") { + continue + } + path := filepath.Join(directory, entry.Name()) + contents, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read %s: %w", path, err) + } + text := strings.ToLower(string(contents)) + if !looksLikeTestWorkflow(text, language, name) { + continue + } + + relativePath, err := filepath.Rel(repositoryRoot, path) + if err != nil { + return nil, nil, fmt.Errorf("make workflow path relative: %w", err) + } + relativePath = filepath.ToSlash(relativePath) + workflows = append(workflows, relativePath) + isConfigured, err := allTestJobsConfigured(contents, language, name) + if err != nil { + return nil, nil, fmt.Errorf("parse %s: %w", path, err) + } + if isConfigured { + configured = append(configured, relativePath) + } + } + + sort.Strings(workflows) + sort.Strings(configured) + return workflows, configured, nil +} + +func allTestJobsConfigured(contents []byte, language, name string) (bool, error) { + var workflow struct { + Jobs map[string]struct { + Steps []struct { + Run string `yaml:"run"` + Uses string `yaml:"uses"` + } `yaml:"steps"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(contents, &workflow); err != nil { + return false, err + } + foundTestJob := false + for _, job := range workflow.Jobs { + var commands []string + configured := false + for _, step := range job.Steps { + commands = append(commands, strings.ToLower(step.Run)) + configured = configured || strings.Contains(strings.ToLower(step.Uses), githubAction) + } + if !looksLikeTestJob(strings.Join(commands, "\n"), language, name) { + continue + } + foundTestJob = true + if !configured { + return false, nil + } + } + return foundTestJob, nil +} + +func looksLikeTestJob(commands, language, name string) bool { + markers := []string{name} + switch language { + case "javascript": + markers = append(markers, "npm test", "npm run test", "yarn test", "yarn run test", "pnpm test", "pnpm run test", "bun test", "bun run test") + case "ruby": + markers = append(markers, "bundle exec rake", "rake test", "rails test") + case "python": + markers = append(markers, "tox", "nox") + } + for _, marker := range markers { + if strings.Contains(commands, marker) { + return true + } + } + return false +} + +func looksLikeTestWorkflow(workflow, language, name string) bool { + markers := []string{name, githubAction} + switch language { + case "javascript": + markers = append(markers, "npm test", "npm run test", "yarn test", "yarn run test", "pnpm test", "pnpm run test", "bun test", "bun run test") + case "ruby": + markers = append(markers, "bundle exec rake", "rake test", "rails test") + case "python": + markers = append(markers, "tox", "nox") + } + for _, marker := range markers { + if strings.Contains(workflow, marker) { + return true + } + } + return false +} + +func instructions(language, name string) string { + actionLanguage := language + var bootstrap string + switch language { + case "javascript": + actionLanguage = "js" + bootstrap = javascriptBootstrap + if name == "cypress" { + bootstrap += "\n\n" + cypressBootstrap + } + if name == "cucumber" { + bootstrap += "\n\nFor Cucumber, also set DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED=false on the test step. This avoids a tracer crash on Cucumber Background/Rule nodes; basic reporting is unaffected." + } + case "python": + bootstrap = pythonBootstrap + case "ruby": + bootstrap = rubyBootstrap + } + // Empty inputs override the action's pinned defaults and request the latest release. + tracerSetting := actionLanguage + "-tracer-version: '' # Latest release" + text := strings.NewReplacer("__FRAMEWORK__", name, "__LANGUAGE__", actionLanguage, "__BOOTSTRAP__", bootstrap, "__TRACER_SETTING__", tracerSetting).Replace(gitHubInstructions) + return strings.ReplaceAll(text, "ddtest testdrive", "ddtest testdrive --framework "+name) +} + +const javascriptBootstrap = "Use Node.js 22 or newer. GitHub Actions cannot set NODE_OPTIONS for later steps, so merge this into the existing test step, preserving any current Node options:\n\n```yaml\nenv:\n NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}\n```\n\nThe --import loader is required for Vitest and other ESM tests." + +const pythonBootstrap = "The action exports PYTHONPATH and PYTEST_ADDOPTS=--ddtrace for pytest. Preserve these variables on the existing test step; do not replace its current arguments. Activate the same Python environment used for the tests before the action. If CI uses tox or nox, pass DD_*, PYTHONPATH, and PYTEST_ADDOPTS into the test environment." + +const rubyBootstrap = "The action installs datadog-ci into the CI bundle and exports RUBYOPT=-rbundler/setup -rdatadog/ci/auto_instrument. Preserve RUBYOPT on the RSpec or Minitest step and merge any existing Ruby options. Keep the existing bundle exec or binstub entry point." + +const cypressBootstrap = `Cypress also needs browser-side instrumentation; NODE_OPTIONS alone is insufficient: + +1. In the existing Cypress config, resolve the tracer root from the action's DD_TRACE_PACKAGE value: path.dirname(path.dirname(process.env.DD_TRACE_PACKAGE)). Only enable this configuration when that variable is present. Use import syntax in an ESM config and require in a CommonJS config. +2. Compose the existing setupNodeEvents callback with the module at path.join(tracerRoot, 'ci/cypress/plugin'), loaded with import in ESM or require in CommonJS. Preserve existing event handlers, including after:run and after:spec; do not replace them. +3. In setupNodeEvents, generate a support wrapper under RUNNER_TEMP. Write a literal require of path.join(tracerRoot, 'ci/cypress/support'), followed by a literal require of the existing resolved supportFile (unless it is false). Set the returned config.supportFile to that wrapper. Generating literal absolute imports lets Cypress's browser bundler resolve the isolated tracer without adding it to package.json. +4. Keep the project's existing application startup, browser installation, and cypress run command. Do not use cypress open. + +Use the same pattern for e2e or component configuration, whichever the selected job runs. The local testdrive creates equivalent wrappers inside its session automatically. See https://docs.datadoghq.com/tests/setup/javascript/ for the Cypress plugin contract.` diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go new file mode 100644 index 00000000..e7c41498 --- /dev/null +++ b/internal/onboard/onboard_test.go @@ -0,0 +1,146 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunPrintsJestGitHubInstructions(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + test: + steps: + - run: npm test +`) + + var output bytes.Buffer + t.Chdir(repositoryRoot) + if err := Run(&output); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + for _, expected := range []string{ + "DDTest found javascript, jest, and GitHub Actions.", + ".github/workflows/test.yml", + "datadog/test-visibility-github-action@v3", + "api_key: ${{ secrets.DD_API_KEY }}", + "NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }}", + "ddtest testdrive", + "post every `Open report:` link", + "Ask a human to connect Datadog", + "without sharing the key itself", + } { + if !strings.Contains(output.String(), expected) { + t.Errorf("Run() output does not contain %q:\n%s", expected, output.String()) + } + } + if strings.Index(output.String(), "ddtest testdrive") > strings.Index(output.String(), "Ask a human to connect Datadog") { + t.Fatalf("API key setup must come after the local testdrive:\n%s", output.String()) + } +} + +func TestRunRecognizesExistingGitHubAction(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + test: + steps: + - uses: datadog/test-visibility-github-action@v3 + - run: npm test +`) + + var output bytes.Buffer + t.Chdir(repositoryRoot) + if err := Run(&output); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + if !strings.Contains(output.String(), "already appears in every detected test workflow") { + t.Fatalf("Run() did not recognize the existing setup:\n%s", output.String()) + } + if !strings.Contains(output.String(), "ddtest testdrive") { + t.Fatalf("Run() did not print the next step:\n%s", output.String()) + } + if !strings.Contains(output.String(), "post every `Open report:` link") { + t.Fatalf("Run() did not tell the agent to share the report:\n%s", output.String()) + } +} + +func TestRunRequiresActionInEveryTestJob(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + unit: + steps: + - uses: datadog/test-visibility-github-action@v3 + - run: npm test + integration: + steps: + - run: npm test +`) + var output bytes.Buffer + t.Chdir(repositoryRoot) + if err := Run(&output); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), "already appears in every detected test workflow") { + t.Fatalf("Run() treated a partially configured workflow as complete:\n%s", output.String()) + } +} + +func TestRunTreatsRepositoryRootAsLiteralPath(t *testing.T) { + parent := t.TempDir() + repositoryRoot := filepath.Join(parent, "project[old]") + if err := os.Mkdir(repositoryRoot, 0755); err != nil { + t.Fatal(err) + } + fixture := newJestRepository(t, "name: tests\njobs:\n test:\n steps:\n - run: npm test\n") + if err := os.Rename(filepath.Join(fixture, "package.json"), filepath.Join(repositoryRoot, "package.json")); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(fixture, ".github"), filepath.Join(repositoryRoot, ".github")); err != nil { + t.Fatal(err) + } + t.Chdir(repositoryRoot) + if err := Run(&bytes.Buffer{}); err != nil { + t.Fatalf("Run() failed for literal repository path: %v", err) + } +} + +func TestRunRequiresGitHubJestWorkflow(t *testing.T) { + repositoryRoot := newJestRepository(t, "name: lint\njobs:\n lint:\n steps:\n - run: npm run lint\n") + + t.Chdir(repositoryRoot) + err := Run(&bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "could not find a GitHub Actions workflow that runs jest") { + t.Fatalf("Run() error = %v", err) + } +} + +func newJestRepository(t *testing.T, workflow string) string { + t.Helper() + repositoryRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(repositoryRoot, "package.json"), []byte(`{ + "scripts": {"test": "jest"}, + "devDependencies": {"jest": "30.0.0"} +}`), 0644); err != nil { + t.Fatal(err) + } + workflowDirectory := filepath.Join(repositoryRoot, ".github", "workflows") + if err := os.MkdirAll(workflowDirectory, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workflowDirectory, "test.yml"), []byte(workflow), 0644); err != nil { + t.Fatal(err) + } + return repositoryRoot +} diff --git a/internal/testdrive/command_test.go b/internal/testdrive/command_test.go new file mode 100644 index 00000000..f0052df3 --- /dev/null +++ b/internal/testdrive/command_test.go @@ -0,0 +1,56 @@ +package testdrive + +import ( + "os" + "path/filepath" + "testing" + + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/settings" + "github.com/stretchr/testify/require" +) + +func TestTestdriveUsesFrameworkCommand(t *testing.T) { + old := settings.Get().Command + settings.Get().Command = "" + t.Cleanup(func() { settings.Get().Command = old }) + for _, tc := range []struct { + runner framework.Framework + args []string + }{ + {framework.NewJest(), []string{"jest"}}, + {framework.NewMocha(), []string{"mocha"}}, + {framework.NewVitest(), []string{"vitest", "run"}}, + {framework.NewPlaywright(), []string{"playwright", "test"}}, + {framework.NewCucumber(), []string{"cucumber-js"}}, + } { + t.Run(tc.runner.Name(), func(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + // Neither a custom script nor a package manager lockfile overrides execution. + require.NoError(t, os.WriteFile("package.json", []byte(`{"scripts":{"test":"jest && echo side-effect","unit":"vitest --config custom.ts"}}`), 0644)) + require.NoError(t, os.WriteFile("yarn.lock", nil, 0644)) + command, args := tc.runner.Command() + require.Equal(t, "npx", command) + require.Equal(t, tc.args, args) + }) + } +} + +func TestTestdrivePreservesExplicitCommandArguments(t *testing.T) { + t.Cleanup(func() { settings.Get().Command = "" }) + settings.Get().Command = `npm run smoke -- --config "config with spaces.js"` + command, args := framework.NewMocha().Command() + require.Equal(t, "npm", command) + require.Equal(t, []string{"run", "smoke", "--", "--config", "config with spaces.js"}, args) +} + +func TestTestdriveMinitestRequiresExecutableRailsBinstub(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "bin", "rails"), []byte("#!/bin/sh\n"), 0644)) + command, args := framework.NewMinitest().Command() + require.Equal(t, "bundle", command) + require.Equal(t, []string{"exec", "rake", "test"}, args) +} diff --git a/internal/testdrive/cypress.go b/internal/testdrive/cypress.go new file mode 100644 index 00000000..6173acc6 --- /dev/null +++ b/internal/testdrive/cypress.go @@ -0,0 +1,198 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kballard/go-shellquote" +) + +// prepareCypress wraps configuration inside the session; project config and +// support files are neither edited nor replaced on disk. +func prepareCypress(root, directory, preload, command string, args []string) ([]string, error) { + inspectionArgs := cypressInspectionArgs(root, command, args) + projectRoot := root + if project := optionValue(inspectionArgs, "--project", "-P"); project != "" { + if filepath.IsAbs(project) { + projectRoot = project + } else { + projectRoot = filepath.Join(root, project) + } + } + var config string + clean := make([]string, 0, len(args)+2) + for i := 0; i < len(args); i++ { + if args[i] == "--config-file" || args[i] == "-C" { + if i+1 == len(args) { + return nil, fmt.Errorf("--config-file requires a path") + } + i++ + config = args[i] + continue + } + if strings.HasPrefix(args[i], "--config-file=") || strings.HasPrefix(args[i], "-C=") { + _, config, _ = strings.Cut(args[i], "=") + continue + } + clean = append(clean, args[i]) + } + if config == "" { + config = optionValue(inspectionArgs, "--config-file", "-C") + } + if config == "false" { + config = "" + } + if config == "" { + for _, name := range []string{"cypress.config.ts", "cypress.config.js", "cypress.config.mjs", "cypress.config.cjs"} { + if _, err := os.Stat(filepath.Join(projectRoot, name)); err == nil { + config = name + break + } + } + } + if config != "" && !filepath.IsAbs(config) { + config = filepath.Join(projectRoot, config) + } + testingType := "e2e" + if optionPresent(inspectionArgs, "--component") { + testingType = "component" + } + tracerRoot := filepath.Dir(filepath.Dir(preload)) + values, _ := json.Marshal(map[string]string{"root": projectRoot, "directory": directory, "tracer": tracerRoot, "testingType": testingType}) + configImport := "const originalImport = {};\n" + if config != "" { + encodedConfig, _ := json.Marshal(config) + configImport = "import originalImport from " + string(encodedConfig) + ";\n" + } + encodedPlugin, _ := json.Marshal(filepath.Join(tracerRoot, "ci", "cypress", "plugin")) + wrapper := "import fs from 'node:fs';\nimport path from 'node:path';\n" + configImport + + "import instrumentImport from " + string(encodedPlugin) + ";\nconst options = " + string(values) + ";\n" + cypressWrapper + path := filepath.Join(directory, "cypress.config.ts") + if err := os.WriteFile(path, []byte(wrapper), 0600); err != nil { + return nil, err + } + return append(clean, "--config-file", path), nil +} + +func optionValue(args []string, options ...string) string { + value := "" + for index := 0; index < len(args); index++ { + for _, option := range options { + if args[index] == option && index+1 < len(args) { + value = args[index+1] + index++ + break + } + if candidate, found := strings.CutPrefix(args[index], option+"="); found { + value = candidate + break + } + } + } + return value +} + +func optionPresent(args []string, option string) bool { + for _, arg := range args { + if arg == option || strings.HasPrefix(arg, option+"=") { + return true + } + } + return false +} + +func cypressInspectionArgs(root, command string, args []string) []string { + for _, arg := range append([]string{command}, args...) { + if strings.Contains(strings.ToLower(filepath.Base(arg)), "cypress") { + return args + } + } + base := strings.ToLower(filepath.Base(command)) + if base != "npm" && base != "yarn" && base != "pnpm" && base != "bun" { + return args + } + script := "" + if len(args) > 0 && args[0] == "test" { + script = "test" + } else if len(args) > 1 && (args[0] == "run" || args[0] == "run-script") { + script = args[1] + } + if script == "" { + return args + } + contents, err := os.ReadFile(filepath.Join(root, "package.json")) + if err != nil { + return args + } + var manifest struct { + Scripts map[string]string `json:"scripts"` + } + if json.Unmarshal(contents, &manifest) != nil { + return args + } + expanded, err := shellquote.Split(manifest.Scripts[script]) + if err != nil { + return args + } + if separator := slicesIndex(args, "--"); separator >= 0 { + expanded = append(expanded, args[separator+1:]...) + } + return expanded +} + +func slicesIndex(values []string, value string) int { + for index, candidate := range values { + if candidate == value { + return index + } + } + return -1 +} + +const cypressWrapper = ` +// Cypress starts the config process in the wrapper's directory. Keep relative +// filesystem operations in the customer's config and hooks rooted in the project. +process.chdir(options.root); +export default (async () => { + const original = originalImport.default || originalImport; + const config = { ...original }; + const types = new Set(['e2e', 'component'].filter(type => original[type])); + types.add(options.testingType); + for (const type of types) { + const originalType = original[type] || {}; + const setup = originalType.setupNodeEvents; + config[type] = { ...originalType, async setupNodeEvents(on, resolved) { + // Compose hooks so adding instrumentation cannot discard customer hooks. + const handlers = new Map(); + const collect = (event, handler) => { + if (event === 'task') { handlers.set(event, { ...(handlers.get(event) || {}), ...handler }); return; } + const previous = handlers.get(event); + handlers.set(event, previous ? async (...args) => { + const first = await previous(...args); + const second = await handler(...args); + return second === undefined ? first : second; + } : handler); + }; + if (setup) resolved = { ...resolved, ...((await setup(collect, resolved)) || {}) }; + const support = path.join(options.directory, type + '-support.cjs'); + let contents = 'require(' + JSON.stringify(path.join(options.tracer, 'ci/cypress/support')) + ');\n'; + if (resolved.supportFile) contents += 'require(' + JSON.stringify(path.resolve(resolved.projectRoot || options.root, resolved.supportFile)) + ');\n'; + fs.writeFileSync(support, contents); + resolved.supportFile = support; + const instrument = instrumentImport.default || instrumentImport; + resolved = (await instrument(collect, resolved)) || resolved; + for (const [event, handler] of handlers) on(event, handler); + return resolved; + }}; + } + return config; +})(); +` diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go new file mode 100644 index 00000000..a78d4e06 --- /dev/null +++ b/internal/testdrive/frameworks_integration_test.go @@ -0,0 +1,137 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestPublicFrameworkTestdrives exercises the shipped CLI, real tracers, and +// real test runners. Browser downloads are explicit test setup, never testdrive +// side effects. Run with DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1. +func TestPublicFrameworkTestdrives(t *testing.T) { + if os.Getenv("DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST") == "" { + t.Skip("set DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1") + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) + defer cancel() + binary := filepath.Join(t.TempDir(), "ddtest") + integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + fixtures := []struct { + name, manifest, command string + files map[string]string + }{ + {"jest", `{"scripts":{"test":"jest"},"devDependencies":{"jest":"30.5.1"}}`, "npm test", map[string]string{"one.test.js": `test('adds', () => expect(1+1).toBe(2));`}}, + {"mocha", `{"scripts":{"test":"mocha"},"devDependencies":{"mocha":"11.7.5"}}`, "npm test", map[string]string{"test/one.js": `const assert = require('node:assert'); it('adds', () => assert.equal(1+1,2));`}}, + {"vitest", `{"type":"module","scripts":{"test":"vitest run"},"devDependencies":{"vitest":"3.2.4"}}`, "npm test", map[string]string{"one.test.js": `import {test,expect} from 'vitest'; test('adds', () => expect(1+1).toBe(2));`}}, + {"playwright", `{"scripts":{"test":"playwright test"},"devDependencies":{"@playwright/test":"1.55.1"}}`, "npm test", map[string]string{"one.spec.js": `const {test,expect} = require('@playwright/test'); test('adds', () => expect(1+1).toBe(2));`}}, + {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, + {"cypress", `{"scripts":{"test":"cypress run"},"devDependencies":{"cypress":"15.1.0"}}`, "npm test", map[string]string{"cypress.config.js": `module.exports={e2e:{supportFile:false,setupNodeEvents(on,config){on('task',{answer:()=>42});on('after:run',()=>{require('node:fs').writeFileSync('original-hook.txt','ran');});return config;}}};`, "cypress/e2e/one.cy.js": `it('preserves hooks', () => { cy.task('answer').should('equal',42); });`}}, + {"pytest", "", "python -m pytest", map[string]string{"requirements.txt": "pytest==8.4.2\n", "pytest.ini": "[pytest]\n", "test_one.py": "def test_adds():\n assert 1 + 1 == 2\n"}}, + {"rspec", "", "bundle exec rspec", map[string]string{"Gemfile": "source 'https://rubygems.org'\ngem 'rspec', '3.13.1'\n", "spec/one_spec.rb": "RSpec.describe 'addition' do\n it('adds') { expect(1 + 1).to eq(2) }\nend\n"}}, + {"minitest", "", "bundle exec rake test", map[string]string{"Gemfile": "source 'https://rubygems.org'\ngem 'rake', '13.3.0'\ngem 'minitest', '5.25.5'\n", "Rakefile": "require 'rake/testtask'\nRake::TestTask.new(:test) { |t| t.pattern = 'test/**/*_test.rb' }\n", "test/one_test.rb": "require 'minitest/autorun'\nclass AdditionTest < Minitest::Test\n def test_adds\n assert_equal 2, 1 + 1\n end\nend\n"}}, + } + for _, fixture := range fixtures { + t.Run(fixture.name, func(t *testing.T) { + name := "project space" + if fixture.name == "rspec" || fixture.name == "minitest" { + name = "project" + } // Keep the recorded Ruby fixture path; build failures retain Bundler diagnostics. + root := filepath.Join(t.TempDir(), name) + require.NoError(t, os.MkdirAll(root, 0755)) + integrationCommand(t, ctx, root, nil, "git", "init", "-q") + for name, contents := range fixture.files { + integrationFile(t, root, name, contents) + } + if fixture.manifest != "" { + integrationFile(t, root, "package.json", fixture.manifest) + } + integrationFile(t, root, ".github/workflows/test.yml", "name: tests\non: push\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: "+fixture.command+"\n") + env := []string{} + if fixture.manifest != "" { + integrationCommand(t, ctx, root, env, "npm", "install", "--no-audit", "--no-fund") + } + if fixture.name == "pytest" { + venv := filepath.Join(t.TempDir(), "venv") + integrationCommand(t, ctx, root, nil, "python3", "-m", "venv", venv) + env = append(env, "PATH="+filepath.Join(venv, "bin")+string(os.PathListSeparator)+os.Getenv("PATH")) + integrationCommand(t, ctx, root, env, filepath.Join(venv, "bin", "python"), "-m", "pip", "install", "pytest==8.4.2") + } + if fixture.name == "rspec" || fixture.name == "minitest" { + integrationCommand(t, ctx, root, env, "bundle", "lock") + } + before := map[string]string{} + for _, name := range []string{"package.json", "package-lock.json", "Gemfile", "Gemfile.lock", "requirements.txt", "cypress.config.js"} { + contents, err := os.ReadFile(filepath.Join(root, name)) + if err == nil { + before[name] = string(contents) + } + } + onboard := integrationCommand(t, ctx, root, env, binary, "onboard") + require.Contains(t, onboard, "datadog/test-visibility-github-action@v3") + output := integrationCommand(t, ctx, root, env, binary, "testdrive", "--yes") + require.Contains(t, output, "Test events received.") + require.Contains(t, output, "Open report:") + reports, err := filepath.Glob(filepath.Join(root, ".testoptimization", "testdrive", "*", "report.html")) + require.NoError(t, err) + require.Len(t, reports, 1) + contents, err := os.ReadFile(reports[0]) + require.NoError(t, err) + require.Contains(t, string(contents), "Test events received.") + traffic, err := filepath.Glob(filepath.Join(filepath.Dir(reports[0]), "intake", "*citestcycle.json")) + require.NoError(t, err) + require.NotEmpty(t, traffic) + for name, contents := range before { + if (fixture.name == "rspec" || fixture.name == "minitest") && (name == "Gemfile" || name == "Gemfile.lock") { + continue // bundle add updates Ruby dependency files. + } + after, err := os.ReadFile(filepath.Join(root, name)) + require.NoError(t, err) + require.Equal(t, contents, string(after), name) + } + if fixture.name == "rspec" || fixture.name == "minitest" { + gemfile, err := os.ReadFile(filepath.Join(root, "Gemfile")) + require.NoError(t, err) + require.Contains(t, string(gemfile), "datadog-ci") + require.FileExists(t, filepath.Join(root, "Gemfile.lock")) + } else if _, existed := before["Gemfile.lock"]; !existed { + _, err := os.Stat(filepath.Join(root, "Gemfile.lock")) + require.True(t, os.IsNotExist(err), "project lockfile must not be created") + } + if fixture.name == "cypress" { + hook, err := os.ReadFile(filepath.Join(root, "original-hook.txt")) + require.NoError(t, err) + require.Equal(t, "ran", string(hook)) + } + t.Log(strings.TrimSpace(output)) + }) + } +} + +func integrationFile(t *testing.T, root, name, contents string) { + t.Helper() + path := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, []byte(contents), 0644)) +} + +func integrationCommand(t *testing.T, ctx context.Context, directory string, env []string, name string, args ...string) string { + t.Helper() + command := exec.CommandContext(ctx, name, args...) + command.Dir = directory + command.Env = append(os.Environ(), env...) + output, err := command.CombinedOutput() + require.NoError(t, err, "%s %v:\n%s", name, args, output) + return string(output) +} diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go index 56256dd8..cebec56e 100644 --- a/internal/testdrive/intake/findings.go +++ b/internal/testdrive/intake/findings.go @@ -81,6 +81,7 @@ type SlowSuite struct { type Facts struct { ConfigurationErrors []string EmptyCoverageEntryCount int + MissingCoverage bool TestCount int TestEventCount int CoveredTestCount int @@ -116,6 +117,7 @@ func (s *Server) Facts() (Facts, error) { findings.SlowSuites, findings.SuiteDurationMedian = slowSuites(findings.Tests) findings.TestCount = len(findings.Tests) findings.CoveredTestCount = uniqueCoveredTestCount(tests, coverages) + findings.MissingCoverage = len(tests) > 0 && len(coverages) == 0 && emptyEntries == 0 findings.BroadCoverage, findings.CoveredFilesMedian = analyzeCoverage(tests, coverages, findings.CoverageLevel) findings.ConfigurationErrors, err = s.configurationErrors() return findings, err diff --git a/internal/testdrive/intake/troubleshooting.go b/internal/testdrive/intake/troubleshooting.go new file mode 100644 index 00000000..82b3609b --- /dev/null +++ b/internal/testdrive/intake/troubleshooting.go @@ -0,0 +1,62 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" +) + +const MissingCoverageFinding = "missing-coverage" + +// Advice explains a likely cause of a finding and a concrete next step. +type Advice struct { + Finding string + Title string + Text string + URL string +} + +// Troubleshoot returns advice only when the project provides evidence for it. +func Troubleshoot(repositoryRoot, framework string, facts Facts) []Advice { + if !facts.MissingCoverage || !strings.EqualFold(framework, "cucumber") || !nycAbsent(repositoryRoot) { + return nil + } + return []Advice{{ + Finding: MissingCoverageFinding, + Title: "Cucumber needs nyc for coverage", + Text: "This project does not declare or install nyc locally. Add it with npm install --save-dev nyc, then run cucumber-js through nyc (for example, nyc cucumber-js) and repeat the testdrive.", + URL: "https://docs.datadoghq.com/tests/test_impact_analysis/setup/javascript/", + }} +} + +func nycAbsent(repositoryRoot string) bool { + data, err := os.ReadFile(filepath.Join(repositoryRoot, "package.json")) + if err != nil { + return false + } + var manifest struct { + Dependencies map[string]json.RawMessage `json:"dependencies"` + DevDependencies map[string]json.RawMessage `json:"devDependencies"` + OptionalDependencies map[string]json.RawMessage `json:"optionalDependencies"` + PeerDependencies map[string]json.RawMessage `json:"peerDependencies"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return false + } + for _, dependencies := range []map[string]json.RawMessage{ + manifest.Dependencies, manifest.DevDependencies, manifest.OptionalDependencies, manifest.PeerDependencies, + } { + if _, exists := dependencies["nyc"]; exists { + return false + } + } + _, err = os.Stat(filepath.Join(repositoryRoot, "node_modules", ".bin", "nyc")) + return errors.Is(err, os.ErrNotExist) +} diff --git a/internal/testdrive/intake/troubleshooting_test.go b/internal/testdrive/intake/troubleshooting_test.go new file mode 100644 index 00000000..ae8b17f3 --- /dev/null +++ b/internal/testdrive/intake/troubleshooting_test.go @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tinylib/msgp/msgp" +) + +func TestMissingCoverageFindingRequiresTestEventsAndNoCoveragePayload(t *testing.T) { + events := msgp.AppendMapHeader(nil, 1) + events = msgp.AppendString(events, "events") + events = msgp.AppendArrayHeader(events, 1) + events = appendEvent(events, "test", 10, 20, 100) + + server := serverWithCoverage(t, events) + server.requests = server.requests[:1] + facts, err := server.Facts() + require.NoError(t, err) + require.True(t, facts.MissingCoverage) + + server = serverWithCoverage(t, events, appendCoverage(nil, 10, 20, 100, "src/greet.js")) + facts, err = server.Facts() + require.NoError(t, err) + require.False(t, facts.MissingCoverage) + + server = serverWithCoverage(t, events, appendCoverage(nil, 10, 20, 100)) + facts, err = server.Facts() + require.NoError(t, err) + require.False(t, facts.MissingCoverage) // Empty coverage has its own finding. +} + +func TestTroubleshootCucumberWithoutNYC(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"devDependencies":{"@cucumber/cucumber":"13.2.1"}}`), 0644)) + facts := Facts{MissingCoverage: true} + + advice := Troubleshoot(root, "Cucumber", facts) + require.Len(t, advice, 1) + require.Equal(t, MissingCoverageFinding, advice[0].Finding) + require.Contains(t, advice[0].Text, "npm install --save-dev nyc") + require.Empty(t, Troubleshoot(root, "Cucumber", Facts{})) + require.Empty(t, Troubleshoot(root, "Jest", facts)) + + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"devDependencies":{"@cucumber/cucumber":"13.2.1","nyc":"17.0.0"}}`), 0644)) + require.Empty(t, Troubleshoot(root, "Cucumber", facts)) +} diff --git a/internal/testdrive/javascript.go b/internal/testdrive/javascript.go index ccc2b159..db3939bb 100644 --- a/internal/testdrive/javascript.go +++ b/internal/testdrive/javascript.go @@ -3,6 +3,7 @@ package testdrive import ( "encoding/json" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -66,3 +67,41 @@ func javascriptTracerVersion(preload string) string { } return pkg.Version } + +func currentNodeVersion() string { + output, err := exec.Command("node", "--version").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(output)) +} + +func supportsNodeImport(version string) bool { + version = strings.TrimPrefix(strings.TrimSpace(version), "v") + parts := strings.Split(version, ".") + if len(parts) < 2 { + return false + } + major, majorErr := strconv.Atoi(parts[0]) + minor, minorErr := strconv.Atoi(parts[1]) + if majorErr != nil || minorErr != nil { + return false + } + return major > 18 || major == 18 && minor >= 18 +} + +func (t *Testdrive) javascriptEnvironment(path string) map[string]string { + env := javascriptEnvironment(path) + // ESM instrumentation is needed by Vitest and by ESM test/config files. + version := "" + if t.nodeVersion != nil { + version = t.nodeVersion() + } + if supportsNodeImport(version) { + registerPath := filepath.Join(filepath.Dir(filepath.Dir(path)), "register.js") + if info, err := os.Stat(registerPath); err == nil && info.Mode().IsRegular() { + env["NODE_OPTIONS"] += " --import " + strconv.Quote(absoluteFileURL(registerPath)) + } + } + return env +} diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go new file mode 100644 index 00000000..d40f4c55 --- /dev/null +++ b/internal/testdrive/multiframework_test.go @@ -0,0 +1,209 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "github.com/DataDog/ddtest/internal/settings" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/stretchr/testify/require" +) + +func TestPrepareAllSupportedFrameworks(t *testing.T) { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest", "rspec", "minitest"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + switch name { + case "pytest": + requireWriteFile(t, filepath.Join(root, "pyproject.toml"), "[tool.pytest.ini_options]\n") + case "rspec", "minitest": + requireWriteFile(t, filepath.Join(root, "Gemfile"), "gem '"+name+"'\n") + default: + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"`+name+`"}}`) + } + t.Chdir(root) + run, err := Prepare("latest") + require.NoError(t, err) + var preview bytes.Buffer + run.Preview(&preview) + require.Contains(t, preview.String(), displayName(name)) + _, err = os.Stat(filepath.Join(root, ".testoptimization")) + require.True(t, os.IsNotExist(err), "preview must be read-only") + if name == "cypress" { + return + } // Browser wrapper has its own real-run test. + run.nodeVersion = func() string { return "v20.0.0" } + installer := &fakeTracer{preloadPath: filepath.Join(root, "isolated")} + run.platform = installer + run.projectTracer = "" + run.projectTracer = "" + run.projectTracer = "" + run.projectTracer = "" + executor := &fakeTestdriveExecutor{} + run.executor = executor + run.startIntake = func(string) (localIntake, error) { + return &fakeIntake{url: "http://127.0.0.1:1234", findings: intake.Facts{TestEventCount: 1, TestCount: 1}}, nil + } + var output bytes.Buffer + require.NoError(t, run.Run(t.Context(), &output)) + require.Contains(t, output.String(), displayName(name)+": Passed") + require.Contains(t, output.String(), "Tests with coverage: 0 / 1") + require.Equal(t, "ddtest-testdrive", executor.env["DD_API_KEY"]) + if name == "cucumber" { + require.Equal(t, "true", executor.env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"]) + } + switch run.language { + case "javascript": + require.NotContains(t, executor.env["NODE_OPTIONS"], "--import") + case "python": + require.NotEmpty(t, executor.env["PYTHONPATH"]) + require.Contains(t, executor.env["PYTEST_ADDOPTS"], "--ddtrace") + require.NotContains(t, executor.env, "NODE_OPTIONS") + case "ruby": + require.NotContains(t, executor.env, "BUNDLE_GEMFILE") + require.Contains(t, preview.String(), "Bundler updates Gemfile and Gemfile.lock.") + require.NotContains(t, preview.String(), "It will not change") + require.Contains(t, output.String(), "datadog-ci · installed in project") + require.Contains(t, executor.env["RUBYOPT"], "datadog/ci/auto_instrument") + require.NotContains(t, executor.env, "NODE_OPTIONS") + } + }) + } +} + +func TestJavaScriptEnvironmentImportsExistingRegister(t *testing.T) { + t.Setenv("NODE_OPTIONS", "") + packageDir := filepath.Join(t.TempDir(), "dd-trace") + require.NoError(t, os.MkdirAll(filepath.Join(packageDir, "ci"), 0755)) + register := filepath.Join(packageDir, "register.js") + require.NoError(t, os.WriteFile(register, nil, 0644)) + run := &Testdrive{nodeVersion: func() string { return "v22.0.0" }} + preload := filepath.Join(packageDir, "ci", "init.js") + require.Contains(t, run.javascriptEnvironment(preload)["NODE_OPTIONS"], " --import ") + require.NoError(t, os.Remove(register)) + require.NotContains(t, run.javascriptEnvironment(preload)["NODE_OPTIONS"], "--import") +} + +func TestSupportsNodeImport(t *testing.T) { + for version, want := range map[string]bool{ + "v18.17.1": false, + "v18.18.0": true, + "v20.0.0": true, + "invalid": false, + } { + if got := supportsNodeImport(version); got != want { + t.Errorf("supportsNodeImport(%q) = %v, want %v", version, got, want) + } + } +} + +func TestPrepareRequiresSelectionForMultipleFrameworks(t *testing.T) { + previous := settings.Get().Framework + t.Cleanup(func() { settings.Get().Framework = previous }) + settings.Get().Framework = "" + root := t.TempDir() + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"vitest run","e2e":"playwright test"}}`) + t.Chdir(root) + _, err := Prepare("latest") + require.ErrorContains(t, err, "--framework") + settings.Get().Framework = "vitest" + run, err := Prepare("latest") + require.NoError(t, err) + require.Equal(t, "vitest", run.framework.Name()) + settings.Get().Framework = "unsupported" + _, err = Prepare("latest") + require.ErrorContains(t, err, "unsupported framework") +} + +func TestLanguageEnvironmentsPreserveCustomerOptions(t *testing.T) { + t.Setenv("PYTHONPATH", "/customer/modules") + t.Setenv("PYTEST_ADDOPTS", "-q") + t.Setenv("RUBYOPT", "-W0") + python := (&Testdrive{language: "python"}).environment("/session/python", "http://127.0.0.1:1234", "session") + require.Equal(t, "/session/python"+string(os.PathListSeparator)+"/customer/modules", python["PYTHONPATH"]) + require.Equal(t, "-q --ddtrace", python["PYTEST_ADDOPTS"]) + ruby := (&Testdrive{language: "ruby"}).environment("/session/Gemfile", "http://127.0.0.1:1234", "session") + require.True(t, strings.HasPrefix(ruby["RUBYOPT"], "-W0 ")) + require.NotContains(t, ruby, "BUNDLE_PATH") // Inherit project Bundler configuration. +} + +func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { + root := t.TempDir() + session := t.TempDir() + config := `module.exports={e2e:{supportFile:false}}` + requireWriteFile(t, filepath.Join(root, "custom.cjs"), config) + args, err := prepareCypress(root, session, "/tracer/ci/init.js", "cypress", []string{"run", "--config-file=custom.cjs", "--browser", "chrome"}) + require.NoError(t, err) + require.Equal(t, []string{"run", "--browser", "chrome", "--config-file", filepath.Join(session, "cypress.config.ts")}, args) + contents, err := os.ReadFile(filepath.Join(root, "custom.cjs")) + require.NoError(t, err) + require.Equal(t, config, string(contents)) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.ts")) + require.NoError(t, err) + require.Contains(t, string(wrapper), filepath.Join(root, "custom.cjs")) + _, err = prepareCypress(root, session, "/tracer/ci/init.js", "cypress", []string{"--config-file"}) + require.ErrorContains(t, err, "requires a path") +} + +func TestCypressWrapperReadsPackageScriptProjectAndConfig(t *testing.T) { + root := t.TempDir() + session := t.TempDir() + project := filepath.Join(root, "apps", "web") + require.NoError(t, os.MkdirAll(project, 0755)) + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"cypress run --project apps/web -C custom.ts"}}`) + requireWriteFile(t, filepath.Join(project, "custom.ts"), `export default {}`) + + args, err := prepareCypress(root, session, "/tracer/ci/init.js", "npm", []string{"test", "--"}) + require.NoError(t, err) + require.Equal(t, []string{"test", "--", "--config-file", filepath.Join(session, "cypress.config.ts")}, args) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.ts")) + require.NoError(t, err) + require.Contains(t, string(wrapper), filepath.Join(project, "custom.ts")) + require.Contains(t, string(wrapper), `"root":"`+project+`"`) +} + +func TestCypressWrapperSupportsConfigFileFalseAndDefaultE2E(t *testing.T) { + root := t.TempDir() + session := t.TempDir() + args, err := prepareCypress(root, session, "/tracer/ci/init.js", "cypress", []string{"run", "-C", "false"}) + require.NoError(t, err) + require.Equal(t, []string{"run", "--config-file", filepath.Join(session, "cypress.config.ts")}, args) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.ts")) + require.NoError(t, err) + require.Contains(t, string(wrapper), `types.add(options.testingType)`) + require.Contains(t, string(wrapper), `const originalImport = {}`) +} + +func TestPythonProjectTracerPreservesImportEnvironment(t *testing.T) { + t.Setenv("PYTHONPATH", "/project/helpers") + t.Setenv("PYTEST_ADDOPTS", "-v") + drive := &Testdrive{language: "python"} + env := drive.environment("", "http://127.0.0.1:1234", "session") + if _, changed := env["PYTHONPATH"]; changed { + t.Fatal("project PYTHONPATH overridden", env) + } + if env["PYTEST_ADDOPTS"] != "-v --ddtrace" { + t.Fatal(env) + } +} + +func TestRubyProjectTracerPreservesBundleEnvironment(t *testing.T) { + drive := &Testdrive{language: "ruby"} + env := drive.environment("", "http://127.0.0.1:1234", "session") + for _, key := range []string{"BUNDLE_GEMFILE", "BUNDLE_PATH", "BUNDLE_APP_CONFIG", "BUNDLE_FROZEN", "BUNDLE_WITHOUT"} { + if _, changed := env[key]; changed { + t.Fatal("project Bundler setting overridden", key) + } + } + if !strings.Contains(env["RUBYOPT"], "-rdatadog/ci/auto_instrument") { + t.Fatal(env) + } +} diff --git a/internal/testdrive/python.go b/internal/testdrive/python.go new file mode 100644 index 00000000..785fedd5 --- /dev/null +++ b/internal/testdrive/python.go @@ -0,0 +1,19 @@ +package testdrive + +import ( + "os" + "strings" +) + +func pythonEnvironment(path string) map[string]string { + env := map[string]string{} + if path != "" { + env["PYTHONPATH"] = path + if existing := os.Getenv("PYTHONPATH"); existing != "" { + env["PYTHONPATH"] += string(os.PathListSeparator) + existing + } + } + env["PYTEST_ADDOPTS"] = strings.TrimSpace(os.Getenv("PYTEST_ADDOPTS") + " --ddtrace") + + return env +} diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index 48a418e8..c61e0b4a 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -28,6 +28,7 @@ type reportCard struct { Title string Count int Context string + Advices []intake.Advice Tests []reportTest Suites []reportSuite Coverages []reportCoverage @@ -220,6 +221,12 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo Context: fmt.Sprintf("%d coverage entries had an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.", findings.EmptyCoverageEntryCount), }) } + if findings.MissingCoverage { + model.Cards = append(model.Cards, reportCard{ + Kind: intake.MissingCoverageFinding, Title: "Coverage not reported", Count: 1, + Context: "Test events arrived, but no code coverage was reported. Test Impact Analysis cannot map these tests to changed files.", + }) + } if len(findings.FailedTests) > 0 { model.Cards = append(model.Cards, reportCard{ Kind: "failed", Title: "Failed tests", Context: "Inspect the errors and source behind each failure.", Count: len(findings.FailedTests), @@ -260,6 +267,9 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo }) } count := len(findings.ConfigurationErrors) + findings.EmptyCoverageEntryCount + if findings.MissingCoverage { + count++ + } for _, size := range []int{ len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.SlowSuites), len(findings.BroadCoverage), } { @@ -271,6 +281,13 @@ func buildReport(repositoryRoot string, findings intake.Facts, commandFailed boo if len(findings.ConfigurationErrors) > 0 { model.Summary += " Tracer configuration errors: " + strings.Join(findings.ConfigurationErrors, ", ") + ". Inspect the captured traffic and test output." } + for _, advice := range intake.Troubleshoot(repositoryRoot, info.Framework, findings) { + for i := range model.Cards { + if model.Cards[i].Kind == advice.Finding { + model.Cards[i].Advices = append(model.Cards[i].Advices, advice) + } + } + } return model } diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html index 3cb8a706..bfbe6624 100644 --- a/internal/testdrive/report.html +++ b/internal/testdrive/report.html @@ -546,7 +546,8 @@

--card-accent: var(--red); } .problem-card[data-kind="flaky"], - .problem-card[data-kind="slow"] { + .problem-card[data-kind="slow"], + .problem-card[data-kind="missing-coverage"] { --card-accent: var(--amber); } .card-button { @@ -616,6 +617,17 @@

justify-content: space-between; background: var(--surface); } + .finding-explanation, + .finding-advice { + padding: 16px 20px; + border-top: 1px solid var(--line); + } + .finding-advice { + background: var(--raised); + } + .finding-advice p { + margin: 8px 0; + } h2 { font-size: 16px; margin: 0; @@ -1208,6 +1220,8 @@

Test report

{{ template "icon-arrow-repeat" }} {{ else if eq .Kind "slow" }} {{ template "icon-stopwatch" }} + {{ else if eq .Kind "missing-coverage" }} + {{ template "icon-exclamation-circle" }} {{ else if eq .Kind "error" }} {{ template "icon-exclamation-circle" }} {{ else }} @@ -1234,6 +1248,16 @@

Test report

{{ .Title }} · {{ .Count }}

+ {{ if eq .Kind "missing-coverage" }} +

{{ .Context }}

+ {{ end }} + {{ range .Advices }} + + {{ end }} {{ range .Tests }}
diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index b9d88e0f..38440d0f 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -24,6 +24,7 @@ func TestReportCountsIndividualFindings(t *testing.T) { want string }{ {"single", intake.Facts{TestEventCount: 1, FailedTests: []intake.Test{{Name: "failed"}}}, "1 finding."}, + {"missing coverage", intake.Facts{TestEventCount: 1, TestCount: 1, MissingCoverage: true}, "1 finding."}, {"multiple in one card", intake.Facts{TestEventCount: 2, FailedTests: []intake.Test{{Name: "one"}, {Name: "two"}}}, "2 findings."}, {"empty coverage without events", intake.Facts{EmptyCoverageEntryCount: 2}, "2 findings."}, {"configuration error without events", intake.Facts{ConfigurationErrors: []string{"skippable_tests"}}, "1 finding."}, @@ -50,6 +51,31 @@ func TestReportCountsIndividualFindings(t *testing.T) { } } +func TestReportShowsCucumberCoverageTroubleshootingWithFinding(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"devDependencies":{"@cucumber/cucumber":"13.2.1"}}`), 0644); err != nil { + t.Fatal(err) + } + facts := intake.Facts{TestEventCount: 2, TestCount: 2, MissingCoverage: true} + model := buildReport(root, facts, false, reportRuntime{Framework: "Cucumber"}) + if model.Summary != "1 finding." || len(model.Cards) != 1 || len(model.Cards[0].Advices) != 1 { + t.Fatalf("missing coverage advice: summary=%q cards=%+v", model.Summary, model.Cards) + } + path, err := writeReport(root, t.TempDir(), facts, false, reportRuntime{Framework: "Cucumber"}) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, expected := range []string{"Coverage not reported", "Cucumber needs nyc for coverage", "npm install --save-dev nyc"} { + if !strings.Contains(string(data), expected) { + t.Fatalf("report does not show %q", expected) + } + } +} + func TestReportKeepsPythonAndRubySourceEscaped(t *testing.T) { for _, extension := range []string{".py", ".rb"} { t.Run(extension, func(t *testing.T) { diff --git a/internal/testdrive/ruby.go b/internal/testdrive/ruby.go new file mode 100644 index 00000000..b2deb334 --- /dev/null +++ b/internal/testdrive/ruby.go @@ -0,0 +1,12 @@ +package testdrive + +import ( + "os" + "strings" +) + +func rubyEnvironment(path string) map[string]string { + env := map[string]string{} + env["RUBYOPT"] = strings.TrimSpace(os.Getenv("RUBYOPT") + " -rbundler/setup -rdatadog/ci/auto_instrument") + return env +} diff --git a/internal/testdrive/testdrive.go b/internal/testdrive/testdrive.go index e97244f9..aab0773f 100644 --- a/internal/testdrive/testdrive.go +++ b/internal/testdrive/testdrive.go @@ -14,6 +14,7 @@ import ( "maps" "os" "path/filepath" + "slices" "strings" "time" @@ -54,6 +55,7 @@ type Testdrive struct { installArgs []string executor commandExecutor startIntake func(string) (localIntake, error) + nodeVersion func() string } // Prepare detects the repository and probes the project tracer without writing files. @@ -77,11 +79,6 @@ func Prepare(version string) (*Testdrive, error) { return nil, err } language := detectedPlatform.Name() - switch runner.Name() { - case "jest": - default: - return nil, fmt.Errorf("testdrive does not yet support %s", runner.Name()) - } command, args := runner.Command() label := map[string]string{"javascript": "dd-trace", "python": "ddtrace", "ruby": "datadog-ci"}[language] + "@" + version @@ -103,7 +100,7 @@ func Prepare(version string) (*Testdrive, error) { } return &Testdrive{projectTracer: projectTracer, session: session, installCommand: installCommand, installArgs: installArgs, repositoryRoot: repositoryRoot, framework: runner, language: language, command: command, args: args, platform: detectedPlatform, tracerVersion: version, tracerLabel: label, - executor: &ext.DefaultCommandExecutor{}, startIntake: func(directory string) (localIntake, error) { return intake.Start(directory) }}, nil + executor: &ext.DefaultCommandExecutor{}, startIntake: func(directory string) (localIntake, error) { return intake.Start(directory) }, nodeVersion: currentNodeVersion}, nil } func displayName(name string) string { @@ -133,8 +130,15 @@ func (t *Testdrive) Preview(output io.Writer) { _, _ = fmt.Fprintf(output, " - install %s: %s\n", t.tracerLabel, shellquote.Join(install...)) } + if t.framework.Name() == "cypress" { + _, _ = fmt.Fprintf(output, " - create Cypress config/support wrappers in %s and preserve existing hooks\n", directory) + } _, _ = fmt.Fprintf(output, " - run: %s\n", shellquote.Join(append([]string{command}, args...)...)) _, _ = fmt.Fprintln(output) + if t.language == "ruby" && t.projectTracer == "" { + _, _ = fmt.Fprintln(output, "Bundler updates Gemfile and Gemfile.lock.") + return + } patterns := map[string][]string{ "javascript": {"package.json", "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb"}, "python": {"pyproject.toml", "setup.py", "setup.cfg", "requirements*.txt", "requirements*.in", "Pipfile", "Pipfile.lock", "poetry.lock", "uv.lock", "pdm.lock", "pylock.toml"}, @@ -185,6 +189,10 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { } tracerLabel := t.tracerLabel + if t.language == "ruby" { + // Bundler owns the project dependency selection. + tracerLabel = "datadog-ci · installed in project" + } if installation.Project { tracerLabel = t.installedTracerLabel(t.projectTracer) + " · reused" } @@ -203,6 +211,15 @@ func (t *Testdrive) Run(ctx context.Context, output io.Writer) (runErr error) { _, _ = fmt.Fprintf(output, "Running %s...\n", shellquote.Join(append([]string{command}, args...)...)) env := t.environment(installation.Path, server.URL(), session.ID()) maps.Copy(env, installation.Env) + if t.framework.Name() == "cypress" { + if command == "npm" && !slices.Contains(args, "--") { + args = append(slices.Clone(args), "--") + } + args, err = prepareCypress(t.repositoryRoot, session.Directory(), installation.Path, command, args) + if err != nil { + return err + } + } testOutput, testErr := t.executor.CombinedOutput(ctx, command, args, env) @@ -300,6 +317,10 @@ func writeFindings(output io.Writer, findings intake.Facts) { _, _ = fmt.Fprintf(output, "Tracer configuration errors: %s. Inspect the captured traffic and test output.\n", strings.Join(findings.ConfigurationErrors, ", ")) } count := len(findings.ConfigurationErrors) + if findings.MissingCoverage { + count++ + _, _ = fmt.Fprintln(output, "Coverage not reported: test events arrived, but no code coverage was reported. Test Impact Analysis cannot map these tests to changed files.") + } if findings.EmptyCoverageEntryCount > 0 { count += findings.EmptyCoverageEntryCount _, _ = fmt.Fprintf(output, "Tracer error: received %d coverage entries with an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.\n", findings.EmptyCoverageEntryCount) @@ -403,8 +424,13 @@ func testEnvironment(intakeURL, sessionID string) map[string]string { func (t *Testdrive) environment(path, intakeURL, sessionID string) map[string]string { env := testEnvironment(intakeURL, sessionID) - if t.language == "javascript" { - maps.Copy(env, javascriptEnvironment(path)) + switch t.language { + case "javascript": + maps.Copy(env, t.javascriptEnvironment(path)) + case "python": + maps.Copy(env, pythonEnvironment(path)) + case "ruby": + maps.Copy(env, rubyEnvironment(path)) } return env }