diff --git a/Makefile b/Makefile index a63e19a2..9deb7588 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ .DEFAULT_GOAL := build -.PHONY: clean fmt vet lint build run release +.PHONY: clean fmt vet lint test build install run release VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X github.com/DataDog/ddtest/internal/buildinfo.Version=$(VERSION) +GOBIN ?= $(shell go env GOBIN) +ifeq ($(strip $(GOBIN)),) +GOBIN := $(shell go env GOPATH)/bin +endif clean: go clean -i -x fmt: @@ -14,6 +18,9 @@ test: go test ./... build: test lint go build -ldflags="$(LDFLAGS)" -o ddtest main.go +install: + GOBIN="$(GOBIN)" go install -ldflags="$(LDFLAGS)" . + @echo "Installed ddtest to $(GOBIN)/ddtest" run: go run main.go release: diff --git a/README.md b/README.md index 13e36fbe..274abde0 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,60 @@ Currently supported: - Python with pytest. - JavaScript with Cucumber, Cypress, Jest, Mocha, Playwright, or Vitest. +## Try Test Optimization locally + +From a supported repository with its test dependencies installed: + +```sh +ddtest onboard +ddtest testdrive +``` + +`onboard` prints GitHub Actions setup instructions. `testdrive` previews its +commands, asks for confirmation, and runs your own tests against a local intake. +It needs no Datadog account, API key, or Agent. Non-interactive callers can review +the preview, then use `ddtest testdrive --yes`. + +All nine frameworks listed above are supported. If a repository contains several +runners, select one with `--framework`. Use `--command` to select a custom entry +point or a small representative part of a large suite: + +```sh +ddtest onboard --framework playwright +ddtest testdrive --framework playwright --command 'npm run test:e2e -- --project=chromium' --yes +``` + +The terminal links to a self-contained HTML report, decoded JSON traffic, and +complete test output under `.testoptimization/testdrive//`. Reports +separate instrumentation success from failed tests, and explicitly indicate when +coverage was not reported. Keep this directory out of source control. Each run +has its own installation, files, and loopback port. + +Local testdrive prerequisites: + +- JavaScript: Node.js 22+, npm for the isolated `dd-trace@6.15.0` installation, + and the project's package manager and dependencies. Vitest/ESM loading and + Cypress config/support wrappers are supplied automatically. +- Python: an activated project environment with pytest and pip. `ddtrace==4.15.1` + is installed into a session-owned directory; the active environment is unchanged. +- Ruby: Ruby/Bundler and native gem build tools. A session-owned bundle adds + `datadog-ci@1.39.0`, preserving the original Gemfile and lockfile. The pinned + tracer's native extensions require a checkout path without spaces. +- Browser suites: install the project's browsers and start any required services + first, or use its existing test command that manages them. Testdrive does not + install browsers or start applications on its own. + +Project dependency manifests and lockfiles are not edited by testdrive. Existing +package scripts and application hooks still run as usual. Tracer downloads require +network access. This release covers root projects and GitHub Actions onboarding; +monorepo orchestration and other CI providers are outside this scope. + +See the [Milestone 2 validation record](docs/testing/onboarding-milestone-2.md) +for tested repository commits, commands, and limitations. + ## Prerequisites -Before using DDTest, you must have **Datadog Test Optimization** already set up and enabled with a Datadog Test Optimization library for your language and framework. DDTest relies on this integration to discover your tests and plan test execution accordingly. +Before using `ddtest plan` or `ddtest run`, you must have **Datadog Test Optimization** already set up and enabled with a Datadog Test Optimization library for your language and framework. DDTest relies on this integration to discover your tests and plan test execution accordingly. Minimum supported library and runtime requirements: diff --git a/docs/design/agent-driven-onboarding.md b/docs/design/agent-driven-onboarding.md new file mode 100644 index 00000000..f2989e2a --- /dev/null +++ b/docs/design/agent-driven-onboarding.md @@ -0,0 +1,231 @@ +# Agent-driven onboarding + +Status: Milestones 0, 1, and 2 implemented; Milestones 3 and 4 proposed + +Last updated: 2026-09-22 + +## Goal + +A user should be able to give a coding agent one prompt: + +> Onboard test optimization using ddtest. + +The agent discovers the flow through `ddtest help`, makes the small CI edit suggested by `ddtest onboard`, runs `ddtest testdrive`, and gives the user a clickable report showing their own tests. + +The local testdrive requires no Datadog account, API key, or Agent. Credentials come last, are configured by a human in CI, and are never shown to the coding agent. + +## Product rules + +- Use the customer's real test suite, not a demo. +- Keep the main flow to `ddtest onboard` and `ddtest testdrive`. +- Detect the repository instead of asking setup questions. +- Show commands and filesystem changes before running them. +- Keep tracer installation isolated from project manifests and lockfiles. +- Explain what worked, what did not, and the next useful action. +- Give the user clickable local and CI links. +- Dogfood a thin end-to-end slice before adding abstractions. + +For now, do not build a generic onboarding framework, YAML editor, public result schema, compatibility matrix, or exhaustive error taxonomy. + +## What exists today + +The working preview is in draft PR #128 on `anmarchenko/agentic-onboarding-runbook`. + +### User flow + +- `ddtest help` points to `ddtest onboard`. +- `ddtest onboard` detects all nine supported platform/framework pairs and candidate GitHub Actions test workflows; the coding agent inspects the actual jobs. Repositories with several frameworks use `--framework`. +- It prints repository-owned Markdown instructions containing the concrete workflow edit, asks the agent to run `ddtest testdrive`, and tells the agent to post every report link to the user. +- The local testdrive comes before the human API-key and GitHub-secret steps. +- `ddtest testdrive` previews its commands and file changes. Interactive terminals ask once for confirmation. Non-interactive callers must review the preview and rerun with `--yes`. +- `make install` builds and installs DDTest into the current user's Go bin directory on macOS and Linux. + +### Local testdrive + +- Each run has a unique directory and kernel-assigned loopback port, so sessions can run concurrently. +- Pinned tracers (`dd-trace@6.15.0`, `ddtrace==4.15.1`, and `datadog-ci@1.39.0`) are installed inside each session. Python keeps the active interpreter; Ruby uses an overlay bundle and a copy of the project lockfile. Project dependency files are untouched. +- The local intake supports the endpoints exercised by that tracer and enables Test Optimization, coverage, Intelligent Test Runner, Early Flake Detection, Auto Test Retries, Impacted Tests, failed-test replay, and Test Management. +- Test events and test- or suite-level coverage are decoded. Raw multipart or msgpack payloads are not retained; saved traffic is JSON only. +- Complete test output is saved separately. + +The terminal and self-contained HTML report answer: + +- Did any tests fail? +- Are any tests flaky because they passed on retry? +- Are any tests unusually slow compared with the median? +- Do any tests or suites cover unusually many files compared with the median? + +Problem cards appear only when the answer is yes. Affected tests are always visible and expand to show attempts, timings, errors, retry information, source excerpts, and coverage. Flaky tests use `test.final_status` and are not also reported as failed. Covered files appear one per line and are paginated 50 at a time. Separate paginated tabs list all suites and tests. + +The report also shows event and coverage counts, framework result, tracer version, isolated installation, saved JSON traffic, and test output. `testdrive` prints it as an absolute clickable `file://` link. + +### Code map + +- `internal/onboard/`: repository detection and embedded Markdown instructions. +- `internal/testdrive/`: session lifecycle, preview, execution, terminal output, and HTML report. +- `internal/testdrive/tracer/`: isolated tracer installation behind the `Tracer` interface. +- `internal/testdrive/intake/`: local intake, JSON capture, event and coverage decoding, and findings. +- `internal/platform/` and `internal/framework/`: existing detection and test commands that future milestones should reuse. + +### Evidence and limits + +- Integration tests run the public CLI with real pinned tracers against fixtures for all nine frameworks. Coverage is reported only when supplied by the tracer. +- A concurrent integration test proves port, traffic, and file isolation. +- Unit tests cover decoding, final status, flaky tests, both coverage granularities, medians, source excerpts, HTML rendering, and confirmation behavior. +- Dogfooding on React Native Paper recognized 1,363 events and coverage for all 680 logical tests, including Early Flake Detection retries. + +Public support covers all nine pairs in Milestone 2, with GitHub Actions onboarding. The intake is not a complete Datadog backend. There is no upload service, Datadog forwarding, stable JSON contract, or savings calculation. Onboarding instructions are copied from the onboarding MCP source rather than shared with it. + +Every change must pass: + +```shell +make test +make lint +``` + +## Milestone 0: prove the local loop — complete + +Milestone 0 proved the risky path before designing the product around it: + +- start a minimal local Test Optimization intake on port `0`; +- install and run a real pinned JavaScript tracer; +- receive a real Jest test event and coverage; +- save each run in its own session directory; +- run two sessions concurrently without collisions; +- require no Datadog credentials or Agent. + +## Milestone 1: delightful Jest preview — implemented + +Milestone 1 turned the spike into the current `onboard` and `testdrive` flow described above. + +Its important interaction contract is: + +- detection and preview happen before any write or external command; +- `ddtest testdrive --yes` is the explicit non-interactive path; +- running the command is one decision—there is no persisted plan, checksum, approval file, or second execution command; +- instrumentation success is independent of whether customer tests pass; +- the project manifest and lockfile are never changed; +- the agent posts the report link to the user. + +Keep dogfooding Jest while later milestones are built. Fix repeated real problems directly; extract shared types only when another working implementation needs them. + +## Milestone 2: every supported platform/framework pair — implemented + +Extend the same basic onboarding and testdrive to the pairs already supported by DDTest: + +| Platform | Frameworks | +| --- | --- | +| JavaScript | Jest, Mocha, Cypress, Playwright, Cucumber, Vitest | +| Python | pytest | +| Ruby | RSpec, Minitest | + +Implemented in vertical slices: + +1. Reuse each platform and framework's `Detect` method and test command. Remove Jest-specific names from the shared report. +2. Add the remaining JavaScript frameworks using the existing isolated `dd-trace` installation. +3. Add one pinned isolated `ddtrace` installation for pytest. +4. Add one pinned isolated Ruby tracer installation shared by RSpec and Minitest. +5. Add a tiny real-tracer fixture for every pair and dogfood at least one real repository per platform. + +For every pair, `onboard` finds the relevant GitHub Actions job and prints one small setup. `testdrive` previews and runs the repository's real test entry point, treats received events as proof even when tests fail, reports missing coverage honestly, and preserves the same terminal and HTML experience where the tracer supplies the data. + +Do not solve monorepos, new CI providers, or cross-platform tracer abstractions here. One known-good tracer version per platform is enough. + +All nine pairs have completed the credential-free flow with JSON traffic and local reports. Manifests and lockfiles are checked for preservation. The validation record distinguishes full-suite runs from representative browser/RSpec subsets and records the pinned Cucumber tracer workaround. + +## Milestone 3: guided Test Parallelization onboarding + +Starting prompt: + +> Onboard test parallelization using ddtest. + +Add `ddtest onboard parallelization` as the obvious next step after Test Optimization works. If Test Optimization is not configured, point back to `ddtest onboard`; do not combine both migrations. + +The command detects the platform, framework, and GitHub Actions test job, then tells the coding agent how to make one concrete transformation using the existing product: + +1. A plan job installs the project as CI already does and runs `ddtest plan`. +2. The plan job exposes DDTest's generated matrix and uploads `.testoptimization/`. +3. A matrix job downloads the artifact and runs `ddtest run --ci-node ${{ matrix.ci_node_index }}`. +4. The old command is removed so CI does not run the full suite twice. + +Start with one worker per CI node, `fail-fast: false`, explicit minimum and maximum parallelism, and the existing CI-job overhead model. Preserve runtime setup, environment, services, caches, permissions, timeouts, and artifacts. DDTest prints instructions; the agent edits the workflow. Do not build a YAML rewriting engine or another planner. + +The first real GitHub Actions run is the test drive. The agent gives the user its link and reports: + +- whether the plan job and every node passed; +- selected node count; +- estimated full-suite and parallel wall time; +- modeled CI overhead and imbalance; +- dedicated slow-suite runners, if any; +- the smallest corrective edit for a concrete setup failure. + +Selecting one node is a valid success when extra nodes would not help enough. Support the same nine pairs as Milestone 2. Stop clearly when an existing matrix or parallel runner cannot be combined safely. + +Milestone 3 ships when an agent can discover the flow from the starting prompt, make a reviewable GitHub Actions edit, run the existing planner and runner, and return a clickable CI link with the important plan facts. + +## Milestone 4: `dd-trace-js` runbook parity + +Match the useful conclusions of the validation runbook, not its internal architecture. Keep `ddtest onboard` and `ddtest testdrive`; do not copy its manifest, execution-plan, checksum, approval-file, persisted-lock, or exit-code machinery. + +The terminal and local report show five independent conclusions: + +- **Basic Reporting:** the tracer reports a real project test. +- **CI configuration:** the selected job visibly initializes Test Optimization and configures transport. +- **Early Flake Detection:** a new passing test is retried with the expected reason. +- **Auto Test Retries:** a fail-once test passes on retry with the expected reason. +- **Test Management:** a configured test is matched and tagged as quarantined. + +Each conclusion is simply works, needs attention, or could not be checked. Name the exact missing prerequisite, first useful action, and cleanup status. Keep this validation scope separate from code-coverage counts. + +Implementation order: + +1. Add all five conclusions to the simplest Jest repository and dogfood the complete flow before generalizing it. +2. Keep the normal happy path: the customer's instrumented suite proves Basic Reporting. +3. After Basic Reporting succeeds, run small DDTest-owned tests for the three advanced features. The local intake supplies settings, known tests, and managed tests; emitted attempts and events must prove behavior. +4. If Basic Reporting is inconclusive, compare one representative test without and with instrumentation and debug logging. Do not double every successful testdrive. +5. Audit the selected GitHub Actions job without executing it. Resolve the setup DDTest generates, direct commands, and simple local package scripts; report dynamic or remote wrappers as inconclusive. +6. Extend the proven Jest slice to Mocha, Cypress, Playwright, Cucumber, and Vitest. + +Temporary tests appear in the preview, are created only after confirmation, and are removed after the run. Their decoded JSON events and output remain in the session. Browser- or application-backed checks may be inconclusive with the missing prerequisite named; DDTest does not start applications or install browsers implicitly. + +Python and Ruby retain Milestone 2's Basic Reporting and suite analysis until their tracer behavior and real dogfood cases justify equivalent advanced checks. + +Milestone 4 ships when all six JavaScript frameworks report the five conclusions without Datadog credentials, preserve project dependencies and concurrent-session isolation, clean up temporary tests, and provide clickable local and CI links. + +## Next work + +Milestone 2 now has public-CLI real-tracer fixtures for every pair and independent open-source dogfood runs. See [the validation record](../testing/onboarding-milestone-2.md). The next planned feature is Milestone 3; keep the existing nine-pair flow green. + +For each slice: + +1. add one real-tracer fixture; +2. run it in a real repository; +3. record what the human or agent had to guess; +4. fix observed friction; +5. run `make test`, `make lint`, and the installed binary. + +## Later ideas + +These should not delay Milestones 2–4: + +- Homebrew distribution. +- One offline source for the Markdown instructions currently duplicated with the onboarding MCP implementation in `dd-source`. +- A broad `ddeval` suite built from real repository and CI shapes. +- Fully local TIA backed by SQLite coverage history, considering committed, staged, unstaged, and untracked changes. +- Real Datadog mode when `DD_API_KEY` is present. +- Local reproduction of CI operating-system and runtime tags. +- Historical test analysis through the Datadog API. +- `ddtest doctor` as a reusable diagnostic command if the integrated flow proves it is needed. +- More CI providers and monorepo orchestration. +- Multiple tracer-version support. +- Local savings estimates and historical replay. +- A hosted Testdog page for sharing a report without a Datadog account. + +Choose the next slice from what users struggle with after Milestone 4, not from a speculative architecture. + +## References + +- `~/p/shepherd/tools/mockdog`: local Test Optimization intake precedent. +- `~/p/dd-trace-js/ci/runbook.md`: validation conclusions and JavaScript adapter behavior. +- `~/p/test-visibility-install-script`: isolated JavaScript tracer installation precedent. +- The onboarding MCP instructions in `~/dd/dd-source`: current source material for CI setup instructions. diff --git a/docs/testing/onboarding-milestone-2.md b/docs/testing/onboarding-milestone-2.md new file mode 100644 index 00000000..b57dca15 --- /dev/null +++ b/docs/testing/onboarding-milestone-2.md @@ -0,0 +1,181 @@ +# Milestone 2 validation — 2026-09-22 + +All nine supported frameworks completed the public `onboard → testdrive --yes → +report.html` flow on independent open-source repositories. Both commands exited +zero for every row below. Each run received real tracer events, retained decoded +JSON traffic and test output, and generated a local HTML report. + +This was a manually selected open-source sample, not a statistically random +compatibility study. Browser and RSpec checks used representative subsets as +identified below. These results do not imply every version or configuration of +each framework is compatible. + +## Environment and checks + +- macOS arm64; Go 1.26.5; Node.js 24.14.0; Python 3.11; Ruby 3.4.7. +- Pinned tracers: `dd-trace@6.15.0`, `ddtrace==4.15.1`, `datadog-ci@1.39.0`. +- `make test` and `make lint` pass. +- The opt-in public CLI matrix runs real fixtures for all nine frameworks, + including paths containing spaces for JavaScript/Python, existing Ruby + lockfiles, a Cucumber Background, and preserved Cypress task/after-run hooks. +- Existing instrumented Jest and concurrent-session isolation checks pass. +- Root manifests and lockfiles were hashed immediately before and after each + open-source run, including presence/absence. All nine snapshots were unchanged. + Dependency installation and application builds happened before those snapshots. +- No Datadog credentials or Agent were used. This validates local instrumentation + and generated onboarding guidance, not a connection to a live Datadog account. + +## Open-source results + +“Covered” means a test had associated coverage in received traffic. Zero means +coverage was not reported, not that the test covered no application code. Events +can exceed logical tests because the local intake enables retries. + +| Framework | Repository / exact commit | Scope | Logical tests | Events | Covered | +| --- | --- | --- | ---: | ---: | ---: | +| jest | [typestack/class-validator](https://github.com/typestack/class-validator/tree/2e1a5c27dbd65b80e27fe96b49bd6e6641fa3603) | Full Jest suite | 743 | 1549 | 743 | +| mocha | [omichelsen/compare-versions](https://github.com/omichelsen/compare-versions/tree/98e81116ef4197b42dca8c3fde8d1e8166e2a81f) | Full Mocha suite | 336 | 678 | 0 | +| vitest | [unjs/destr](https://github.com/unjs/destr/tree/541b6f9aeada9fc30de9c5a7e086dbfc1c6fcdc7) | Full Vitest suite (lint omitted) | 22 | 44 | 22 | +| playwright | [dropbox/ttvc](https://github.com/dropbox/ttvc/tree/239481cdecb841f63f0b3f3e13be52275333b18d) | One Chromium browser test | 1 | 2 | 0 | +| cypress | [cypress-io/cypress-example-kitchensink](https://github.com/cypress-io/cypress-example-kitchensink/tree/ddaaa92080b68d71d7a1797b4ed20ada18ed2a2a) | Six Todo browser tests | 6 | 12 | 0 | +| cucumber | [cloudevents/sdk-javascript](https://github.com/cloudevents/sdk-javascript/tree/bf5d53f2862248d72d9869cf5173f51b32583faf) | HTTP/Kafka conformance suite | 7 | 14 | 0 | +| pytest | [pallets/itsdangerous](https://github.com/pallets/itsdangerous/tree/672971d66a2ef9f85151e53283113f33d642dabd) | Full pytest suite | 297 | 297 | 297 | +| rspec | [ruby-concurrency/concurrent-ruby](https://github.com/ruby-concurrency/concurrent-ruby/tree/e674fb2688206bb7cb66dee108e1d08184413afb) | AtomicBoolean spec file | 25 | 25 | 25 | +| minitest | [ruby-i18n/i18n](https://github.com/ruby-i18n/i18n/tree/547917dd8d41fab781a81880f22687fc4eac5d85) | Full Minitest suite | 1607 | 1607 | 1607 | + +## Reproduction + +Build DDTest with Go 1.26.5 and put the resulting binary on PATH. Check out each +linked commit and install that project's dependencies first. Use Node.js 22+ for +the JavaScript tracer and activate the project's Python environment for pytest. +The Ruby checks use Ruby 3.4.7 with native extension build tools. + +Repository preparation used: + +- class-validator and compare-versions: `npm ci --no-audit --no-fund`. +- destr: `pnpm install --frozen-lockfile` (pnpm 10); the run selects Vitest directly. +- ttvc: `npm install --no-audit --no-fund`, `npm run build`, and + `node_modules/.bin/playwright install chromium`. The existing Playwright config + starts its application through `yarn express`. The npm preparation updated its + Yarn lockfile; the testdrive itself left the prepared dependency files unchanged. +- Cypress Kitchen Sink: `npm ci --no-audit --no-fund`, then `npm start` on port 8080. + Browser installation is part of project preparation, not a testdrive operation. +- CloudEvents: `git submodule update --init --depth 1`, `npm ci --no-audit --no-fund`, + `npm run build:schema`, then `npm run build:src`. Its conformance submodule was + `eddc279339609ed92d128bcd2b0d5c558a7ce396`. +- itsdangerous: create/activate a Python 3.11 venv and run + `python -m pip install -e . pytest freezegun`. +- concurrent-ruby and i18n: their original Gemfiles were evaluated by the isolated + bundle; no project bundle or manifest changes were needed. + +Run `ddtest onboard --framework FRAMEWORK` in each repository, followed by: + +**typestack/class-validator** + +```sh +ddtest testdrive --framework jest --yes +``` + +**omichelsen/compare-versions** + +```sh +ddtest testdrive --framework mocha --yes +``` + +**unjs/destr** + +```sh +ddtest testdrive --framework vitest --yes --command 'node_modules/.bin/vitest run' +``` + +**dropbox/ttvc** + +```sh +ddtest testdrive --framework playwright --yes --command 'node_modules/.bin/playwright test test/e2e/stylesheet1 --project=chromium --workers=1' +``` + +**cypress-io/cypress-example-kitchensink** + +```sh +ddtest testdrive --framework cypress --yes --command 'node_modules/.bin/cypress run --spec cypress/e2e/1-getting-started/todo.cy.js' +``` + +**cloudevents/sdk-javascript** + +```sh +ddtest testdrive --framework cucumber --yes --command 'npm run conformance' +``` + +**pallets/itsdangerous** + +```sh +ddtest testdrive --framework pytest --yes +``` + +**ruby-concurrency/concurrent-ruby** + +```sh +ddtest testdrive --framework rspec --yes --command 'bundle exec rspec spec/concurrent/atomic/atomic_boolean_spec.rb' +``` + +**ruby-i18n/i18n** + +```sh +ddtest testdrive --framework minitest --yes +``` + +Each run prints an absolute `Open report:` link. The report, `test-output.txt`, +and decoded `intake/*.json` files remain under its unique +`.testoptimization/testdrive//` directory. + +Run the reproducible fixture matrix from DDTest itself: + +```sh +GOTOOLCHAIN=go1.26.5 make test +GOTOOLCHAIN=go1.26.5 make lint +GOTOOLCHAIN=go1.26.5 \ + DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1 \ + DDTEST_RUN_NPM_INTEGRATION_TEST=1 \ + go test ./internal/testdrive \ + -run 'TestPublicFrameworkTestdrives|TestInstrumentedJestFixture' \ + -count=1 -v +``` + +The integration matrix installs project dependencies in temporary directories, +including Cypress's browser. Python, pip/venv, Ruby, Bundler, Node, npm, and native +gem build tools must already be installed. + +## Problems found and addressed + +- Python's `uv run tox` and Ruby's `bundle exec rake` workflows were initially + missed. They are now included as candidate workflows for the coding agent to + inspect; this does not claim to statically validate every CI job. +- The pinned JavaScript tracer dereferenced a missing `scenario.id` for Cucumber + Background nodes when impacted-test detection was enabled. Local Cucumber runs + disable that feature, and generated CI guidance carries the same workaround. + CloudEvents' conformance scenarios and the independent + [Cucumber 7 TypeScript starter](https://github.com/hdorgeval/cucumber7-ts-starter/tree/02fb70d50ce0ccbad7ccc51fc856362f5efc5470) + passed after this change. The starter has no GitHub Actions workflow, so its + extra check covered testdrive only. A Background is now in the fixture matrix. +- Returning a completely empty known-tests map disabled Jest's existing retry + behavior. The intake now supplies empty datasets for all supported runner names; + the existing Jest retry and concurrent-session assertions remain intact. +- Cypress starts its config process beside the generated wrapper. The wrapper now + restores the project working directory before loading the original config, so + relative file access and existing after-run hooks keep their original behavior. + The fixture asserts both a customer task and an after-run file write. +- Ruby's native tracer extension failed to compile under a path containing spaces. + Testdrive now reports this prerequisite directly. Use a checkout without spaces + for Ruby; JavaScript and Python fixtures cover paths containing spaces. +- Ruby uses a copied lockfile with relocated local PATH sources, preserving the + customer's resolved versions where compatible while adding the pinned tracer + only to the session-owned bundle. + +## Deliberate limits + +Root projects and GitHub Actions onboarding only. No monorepo orchestration, +implicit application/browser installation, live Datadog connection validation, +public result schema, savings estimates, or Milestone 3/4 feature guarantees. +Only macOS arm64 was used for this open-source sample. Coverage depends on the +framework/tracer and the project's existing coverage setup; unsupported or absent +coverage is shown explicitly rather than inferred from passing tests. diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 23b7c71b..dc398ada 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -29,9 +29,17 @@ import ( var defaultParallelism = settings.DefaultParallelism() var rootCmd = &cobra.Command{ - Use: "ddtest", - Short: "A test runner from Datadog", - Long: "Command line tool for running tests with Datadog Test Optimization.", + Use: "ddtest", + Short: "A test runner from Datadog", + Long: `Command line tool for running tests with Datadog Test Optimization. + +Start here: + + cd + ddtest onboard + +The onboard command detects the test and CI setup, explains the required +changes, points to local verification, and tells you when human action is needed.`, Version: buildinfo.CurrentVersion(), PersistentPreRunE: runPersistentPreRun, } @@ -138,6 +146,8 @@ func init() { rootCmd.AddCommand(planCmd) rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(onboardCmd) + rootCmd.AddCommand(testdriveCmd) cobra.OnInitialize(settings.Init) } diff --git a/internal/cmd/cmd_test.go b/internal/cmd/cmd_test.go index 62bd5d2b..ec931b49 100644 --- a/internal/cmd/cmd_test.go +++ b/internal/cmd/cmd_test.go @@ -271,23 +271,17 @@ func TestRootCommandFlags(t *testing.T) { } func TestCommandHierarchy(t *testing.T) { - // Verify that planCmd and runCmd are added to rootCmd + // Verify that the public commands are added to rootCmd. commands := rootCmd.Commands() - var foundPlan, foundRun bool + found := make(map[string]bool) for _, cmd := range commands { - if cmd.Name() == "plan" { - foundPlan = true - } - if cmd.Name() == "run" { - foundRun = true - } + found[cmd.Name()] = true } - if !foundPlan { - t.Error("plan command should be added to root command") - } - if !foundRun { - t.Error("run command should be added to root command") + for _, name := range []string{"onboard", "plan", "run", "testdrive"} { + if !found[name] { + t.Errorf("%s command should be added to root command", name) + } } } @@ -557,8 +551,15 @@ func TestExecute(t *testing.T) { } output := buf.String() - if !strings.Contains(output, "ddtest") { - t.Error("help output should contain command name 'ddtest'") + for _, expected := range []string{ + "Start here:", + "cd ", + "ddtest onboard", + "onboard Start here: onboard this repository to Test Optimization", + } { + if !strings.Contains(output, expected) { + t.Errorf("help output should contain %q:\n%s", expected, output) + } } } @@ -760,8 +761,8 @@ func TestCommandUsage(t *testing.T) { } // Expected commands (cobra adds completion and help automatically) - expectedCommands := []string{"plan", "run"} - requiredCommands := []string{"completion", "help [command]", "plan", "run"} + expectedCommands := []string{"onboard", "plan", "run", "testdrive"} + requiredCommands := []string{"completion", "help [command]", "onboard", "plan", "run", "testdrive"} // Verify minimum expected commands exist for _, expected := range expectedCommands { diff --git a/internal/cmd/onboard.go b/internal/cmd/onboard.go new file mode 100644 index 00000000..ef8597a2 --- /dev/null +++ b/internal/cmd/onboard.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "fmt" + "io" + "os" + + "github.com/DataDog/ddtest/internal/onboard" + "github.com/spf13/cobra" +) + +var onboardCmd = newOnboardCommand(func(root string, output io.Writer) error { + return onboard.RunWithFramework(root, onboardingFramework(), output) +}) + +func onboardingFramework() string { + if rootCmd.PersistentFlags().Changed("framework") { + name, _ := rootCmd.PersistentFlags().GetString("framework") + return name + } + return "" +} + +func newOnboardCommand(run func(string, io.Writer) error) *cobra.Command { + return &cobra.Command{ + Use: "onboard", + Short: "Start here: onboard this repository to Test Optimization", + Long: "Detects the supported test setup and prints the smallest Datadog Test Optimization onboarding instructions. It does not edit files.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + repositoryRoot, err := os.Getwd() + if err != nil { + return fmt.Errorf("find repository root: %w", err) + } + return run(repositoryRoot, cmd.OutOrStdout()) + }, + } +} diff --git a/internal/cmd/onboard_test.go b/internal/cmd/onboard_test.go new file mode 100644 index 00000000..796ae0e5 --- /dev/null +++ b/internal/cmd/onboard_test.go @@ -0,0 +1,40 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "bytes" + "io" + "strings" + "testing" +) + +func TestOnboardCommandRunsDetection(t *testing.T) { + called := false + command := newOnboardCommand(func(repositoryRoot string, output io.Writer) error { + called = true + if strings.TrimSpace(repositoryRoot) == "" { + t.Fatal("repository root is empty") + } + _, _ = io.WriteString(output, "onboarding instructions\n") + return nil + }) + if command.Short != "Start here: onboard this repository to Test Optimization" { + t.Fatalf("Short = %q", command.Short) + } + var output bytes.Buffer + command.SetOut(&output) + + if err := command.ExecuteContext(t.Context()); err != nil { + t.Fatalf("ExecuteContext() unexpected error: %v", err) + } + if !called { + t.Fatal("onboard command did not run detection") + } + if output.String() != "onboarding instructions\n" { + t.Fatalf("output = %q", output.String()) + } +} diff --git a/internal/cmd/testdrive.go b/internal/cmd/testdrive.go new file mode 100644 index 00000000..6f56d5a6 --- /dev/null +++ b/internal/cmd/testdrive.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/DataDog/ddtest/internal/testdrive" + "github.com/spf13/cobra" +) + +type testdriveExecution interface { + Preview(io.Writer) + Run(context.Context, io.Writer) error +} + +type prepareTestdrive func(string) (testdriveExecution, error) + +var testdriveCmd = newTestdriveCommand(func(repositoryRoot string) (testdriveExecution, error) { + return testdrive.PrepareWithFramework(repositoryRoot, onboardingFramework()) +}) + +func newTestdriveCommand(prepare prepareTestdrive) *cobra.Command { + command := &cobra.Command{ + Use: "testdrive", + Short: "Try Test Optimization on the local test suite", + Long: "Runs the detected test suite once with Datadog Test Optimization and a local intake. No Datadog API key is required.", + Args: cobra.NoArgs, + } + command.Flags().Bool("yes", false, "Run after printing the changes and commands") + command.RunE = func(cmd *cobra.Command, _ []string) error { + repositoryRoot, err := os.Getwd() + if err != nil { + return fmt.Errorf("find repository root: %w", err) + } + execution, err := prepare(repositoryRoot) + if err != nil { + return err + } + + output := cmd.OutOrStdout() + execution.Preview(output) + yes, err := cmd.Flags().GetBool("yes") + if err != nil { + return err + } + if !yes { + confirmed, err := confirmTestdrive(cmd.InOrStdin(), output, isTerminal(cmd.InOrStdin())) + if err != nil { + return err + } + if !confirmed { + _, _ = fmt.Fprintln(output, "Testdrive cancelled.") + return nil + } + } + + return execution.Run(commandContext(cmd), output) + } + return command +} + +func confirmTestdrive(input io.Reader, output io.Writer, interactive bool) (bool, error) { + if !interactive { + return false, fmt.Errorf("standard input is not interactive; review the preview and run `ddtest testdrive --yes`") + } + + _, _ = fmt.Fprint(output, "\nContinue? [y/N] (coding agents can rerun with --yes): ") + answer, err := bufio.NewReader(input).ReadString('\n') + if err != nil && err != io.EOF { + return false, fmt.Errorf("read confirmation: %w", err) + } + answer = strings.ToLower(strings.TrimSpace(answer)) + return answer == "y" || answer == "yes", nil +} + +func isTerminal(input io.Reader) bool { + file, ok := input.(*os.File) + if !ok { + return false + } + info, err := file.Stat() + return err == nil && info.Mode()&os.ModeCharDevice != 0 +} diff --git a/internal/cmd/testdrive_test.go b/internal/cmd/testdrive_test.go new file mode 100644 index 00000000..1c504621 --- /dev/null +++ b/internal/cmd/testdrive_test.go @@ -0,0 +1,104 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package cmd + +import ( + "bytes" + "context" + "io" + "strings" + "testing" +) + +type fakeTestdriveExecution struct { + previewed bool + run bool +} + +func (f *fakeTestdriveExecution) Preview(output io.Writer) { + f.previewed = true + _, _ = io.WriteString(output, "preview\n") +} + +func (f *fakeTestdriveExecution) Run(context.Context, io.Writer) error { + f.run = true + return nil +} + +func TestConfirmTestdriveInteractive(t *testing.T) { + var output bytes.Buffer + confirmed, err := confirmTestdrive(strings.NewReader("yes\n"), &output, true) + if err != nil { + t.Fatalf("confirmTestdrive() unexpected error: %v", err) + } + if !confirmed { + t.Fatal("confirmTestdrive() = false, want true") + } + if !strings.Contains(output.String(), "--yes") { + t.Fatalf("prompt does not explain the agent path: %q", output.String()) + } +} + +func TestConfirmTestdriveDefaultsToNo(t *testing.T) { + confirmed, err := confirmTestdrive(strings.NewReader("\n"), &bytes.Buffer{}, true) + if err != nil { + t.Fatalf("confirmTestdrive() unexpected error: %v", err) + } + if confirmed { + t.Fatal("confirmTestdrive() = true, want false") + } +} + +func TestConfirmTestdriveNonInteractiveRequiresYes(t *testing.T) { + confirmed, err := confirmTestdrive(strings.NewReader("yes\n"), &bytes.Buffer{}, false) + if confirmed { + t.Fatal("confirmTestdrive() accepted piped input") + } + if err == nil || !strings.Contains(err.Error(), "ddtest testdrive --yes") { + t.Fatalf("confirmTestdrive() error = %v", err) + } +} + +func TestTestdriveCommandHasYesFlag(t *testing.T) { + if testdriveCmd.Flags().Lookup("yes") == nil { + t.Fatal("testdrive command does not define --yes") + } +} + +func TestTestdriveCommandYesPreviewsAndRuns(t *testing.T) { + execution := &fakeTestdriveExecution{} + command := newTestdriveCommand(func(string) (testdriveExecution, error) { + return execution, nil + }) + var output bytes.Buffer + command.SetOut(&output) + command.SetArgs([]string{"--yes"}) + + if err := command.ExecuteContext(t.Context()); err != nil { + t.Fatalf("ExecuteContext() unexpected error: %v", err) + } + if !execution.previewed || !execution.run { + t.Fatalf("previewed = %v, run = %v", execution.previewed, execution.run) + } +} + +func TestTestdriveCommandNonInteractivePreviewsWithoutRunning(t *testing.T) { + execution := &fakeTestdriveExecution{} + command := newTestdriveCommand(func(string) (testdriveExecution, error) { + return execution, nil + }) + var output bytes.Buffer + command.SetIn(strings.NewReader("yes\n")) + command.SetOut(&output) + + err := command.ExecuteContext(t.Context()) + if err == nil || !strings.Contains(err.Error(), "--yes") { + t.Fatalf("ExecuteContext() error = %v", err) + } + if !execution.previewed || execution.run { + t.Fatalf("previewed = %v, run = %v", execution.previewed, execution.run) + } +} diff --git a/internal/framework/cucumber.go b/internal/framework/cucumber.go index 93402ba6..061f21b5 100644 --- a/internal/framework/cucumber.go +++ b/internal/framework/cucumber.go @@ -86,7 +86,10 @@ func NewCucumber() *Cucumber { func (c *Cucumber) SetPlatformEnv(platformEnv map[string]string) { c.platformEnv = platformEnv } func (c *Cucumber) GetPlatformEnv() map[string]string { return c.platformEnv } func (c *Cucumber) Name() string { return "cucumber" } -func (c *Cucumber) SupportsFullTestDiscovery() bool { return false } +func (c *Cucumber) Detect(repositoryRoot string) (bool, error) { + return detectJavaScriptFramework(repositoryRoot, "@cucumber/cucumber", "cucumber") +} +func (c *Cucumber) SupportsFullTestDiscovery() bool { return false } func (c *Cucumber) SourceFileForSuite(suite string) (string, bool) { suite = utils.NormalizePath(strings.TrimSpace(suite)) diff --git a/internal/framework/cypress.go b/internal/framework/cypress.go index 6a42845a..983e8232 100644 --- a/internal/framework/cypress.go +++ b/internal/framework/cypress.go @@ -62,7 +62,10 @@ func NewCypress() *Cypress { func (c *Cypress) SetPlatformEnv(platformEnv map[string]string) { c.platformEnv = platformEnv } func (c *Cypress) GetPlatformEnv() map[string]string { return c.platformEnv } func (c *Cypress) Name() string { return "cypress" } -func (c *Cypress) SupportsFullTestDiscovery() bool { return false } +func (c *Cypress) Detect(repositoryRoot string) (bool, error) { + return detectJavaScriptFramework(repositoryRoot, "cypress") +} +func (c *Cypress) SupportsFullTestDiscovery() bool { return false } func (c *Cypress) SourceFileForSuite(suite string) (string, bool) { suite = strings.TrimSpace(suite) diff --git a/internal/framework/framework.go b/internal/framework/framework.go index 57c97be4..0242b4c2 100644 --- a/internal/framework/framework.go +++ b/internal/framework/framework.go @@ -2,6 +2,11 @@ package framework import ( "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" "github.com/DataDog/ddtest/internal/discovery" "github.com/DataDog/ddtest/internal/testoptimization" @@ -9,6 +14,7 @@ import ( type Framework interface { Name() string + Detect(repositoryRoot string) (bool, error) TestPattern() string DiscoverTestFiles(ctx context.Context, testFiles discovery.TestFileSet) ([]string, error) DiscoverTests(ctx context.Context, testFiles discovery.TestFileSet) ([]testoptimization.Test, error) @@ -19,3 +25,88 @@ type Framework interface { SourceFileForSuite(suite string) (string, bool) HasUnskippableMarker(testFile string) bool } + +type packageManifest struct { + Scripts map[string]string `json:"scripts"` + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` +} + +func readPackageManifest(repositoryRoot string) (packageManifest, bool, error) { + path := filepath.Join(repositoryRoot, "package.json") + contents, err := os.ReadFile(path) + if os.IsNotExist(err) { + return packageManifest{}, false, nil + } + if err != nil { + return packageManifest{}, false, fmt.Errorf("read %s: %w", path, err) + } + + var manifest packageManifest + if err := json.Unmarshal(contents, &manifest); err != nil { + return packageManifest{}, false, fmt.Errorf("parse %s: %w", path, err) + } + return manifest, true, nil +} + +func (m packageManifest) usesAny(names ...string) bool { + for _, name := range names { + if _, ok := m.Dependencies[name]; ok { + return true + } + if _, ok := m.DevDependencies[name]; ok { + return true + } + } + + for _, script := range m.Scripts { + script = strings.ToLower(script) + for _, name := range names { + if strings.Contains(script, strings.ToLower(name)) { + return true + } + } + } + return false +} + +func detectJavaScriptFramework(repositoryRoot string, packageNames ...string) (bool, error) { + manifest, found, err := readPackageManifest(repositoryRoot) + if err != nil || !found { + return false, err + } + return manifest.usesAny(packageNames...), nil +} + +func detectAnyPath(repositoryRoot string, paths ...string) (bool, error) { + for _, relativePath := range paths { + path := filepath.Join(repositoryRoot, relativePath) + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if !os.IsNotExist(err) { + return false, fmt.Errorf("inspect %s: %w", path, err) + } + } + return false, nil +} + +func detectFileContaining(repositoryRoot string, filename string, values ...string) (bool, error) { + path := filepath.Join(repositoryRoot, filename) + contents, err := os.ReadFile(path) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("read %s: %w", path, err) + } + + text := strings.ToLower(string(contents)) + for _, value := range values { + if strings.Contains(text, strings.ToLower(value)) { + return true, nil + } + } + return false, nil +} diff --git a/internal/framework/framework_test.go b/internal/framework/framework_test.go new file mode 100644 index 00000000..1fd538e1 --- /dev/null +++ b/internal/framework/framework_test.go @@ -0,0 +1,88 @@ +package framework + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFrameworksDetectTheirProjects(t *testing.T) { + tests := []struct { + name string + framework Framework + files map[string]string + dirs []string + }{ + {name: "rspec", framework: NewRSpec(), files: map[string]string{"Gemfile": `gem "rspec"`}}, + {name: "minitest", framework: NewMinitest(), dirs: []string{"test"}}, + {name: "pytest", framework: NewPytest(), files: map[string]string{"pytest.ini": "[pytest]"}}, + {name: "jest", framework: NewJest(), files: map[string]string{"package.json": `{"scripts":{"test":"jest"},"devDependencies":{"jest":"latest"}}`}}, + {name: "mocha", framework: NewMocha(), files: map[string]string{"package.json": `{"devDependencies":{"mocha":"latest"}}`}}, + {name: "cypress", framework: NewCypress(), files: map[string]string{"package.json": `{"devDependencies":{"cypress":"latest"}}`}}, + {name: "playwright", framework: NewPlaywright(), files: map[string]string{"package.json": `{"devDependencies":{"@playwright/test":"latest"}}`}}, + {name: "cucumber", framework: NewCucumber(), files: map[string]string{"package.json": `{"devDependencies":{"@cucumber/cucumber":"latest"}}`}}, + {name: "vitest", framework: NewVitest(), files: map[string]string{"package.json": `{"devDependencies":{"vitest":"latest"}}`}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repositoryRoot := t.TempDir() + detected, err := test.framework.Detect(repositoryRoot) + if err != nil { + t.Fatalf("Detect() unexpected error: %v", err) + } + if detected { + t.Fatal("Detect() = true for an empty repository") + } + + for _, dir := range test.dirs { + if err := os.MkdirAll(filepath.Join(repositoryRoot, dir), 0755); err != nil { + t.Fatal(err) + } + } + for filename, contents := range test.files { + if err := os.WriteFile(filepath.Join(repositoryRoot, filename), []byte(contents), 0644); err != nil { + t.Fatal(err) + } + } + + detected, err = test.framework.Detect(repositoryRoot) + if err != nil { + t.Fatalf("Detect() unexpected error: %v", err) + } + if !detected { + t.Fatal("Detect() = false for a matching repository") + } + }) + } +} + +func TestJestDetectUsesProjectTestScript(t *testing.T) { + repositoryRoot := t.TempDir() + manifest := `{"scripts":{"test":"jest"},"devDependencies":{"jest":"latest"}}` + if err := os.WriteFile(filepath.Join(repositoryRoot, "package.json"), []byte(manifest), 0644); err != nil { + t.Fatal(err) + } + + jest := NewJest() + detected, err := jest.Detect(repositoryRoot) + if err != nil { + t.Fatalf("Detect() unexpected error: %v", err) + } + if !detected { + t.Fatal("Detect() = false") + } + command, args := jest.TestCommand(nil) + if command != "npm" { + t.Fatalf("TestCommand() command = %q, want npm", command) + } + wantArgs := []string{"test", "--", "--runInBand"} + if len(args) != len(wantArgs) { + t.Fatalf("TestCommand() args = %v, want %v", args, wantArgs) + } + for i := range wantArgs { + if args[i] != wantArgs[i] { + t.Fatalf("TestCommand() args = %v, want %v", args, wantArgs) + } + } +} diff --git a/internal/framework/jest.go b/internal/framework/jest.go index b620c552..5265a5d1 100644 --- a/internal/framework/jest.go +++ b/internal/framework/jest.go @@ -57,6 +57,27 @@ func (j *Jest) Name() string { return "jest" } +func (j *Jest) Detect(repositoryRoot string) (bool, error) { + manifest, found, err := readPackageManifest(repositoryRoot) + if err != nil || !found { + return false, err + } + if !manifest.usesAny("jest") { + return false, nil + } + if len(j.commandOverride) > 0 { + return true, nil + } + if strings.TrimSpace(manifest.Scripts["test"]) == "" { + return true, nil + } + + // Use the project's test script so testdrive exercises the same entry point + // that developers and CI normally use. + j.commandOverride = []string{"npm", "test", "--", "--runInBand"} + return true, nil +} + // We will not be discovering tests, but test suites. // We'll be working outside of the Node.js process func (j *Jest) SupportsFullTestDiscovery() bool { @@ -120,10 +141,7 @@ func (j *Jest) DiscoverTestFiles(ctx context.Context, testFiles discovery.TestFi } func (j *Jest) RunTests(ctx context.Context, testFiles []string, envMap map[string]string) error { - command, baseArgs := j.getJestCommand() - args := slices.Clone(baseArgs) - args = append(args, "--runTestsByPath") - args = append(args, testFiles...) + command, args := j.TestCommand(testFiles) slog.Info("Running tests with command", "command", command, "args", args) @@ -133,6 +151,18 @@ func (j *Jest) RunTests(ctx context.Context, testFiles []string, envMap map[stri return j.executor.Run(ctx, command, args, mergedEnv) } +// TestCommand returns the command used to run the selected tests. An empty +// selection runs the framework's full suite. +func (j *Jest) TestCommand(testFiles []string) (string, []string) { + command, baseArgs := j.getJestCommand() + args := slices.Clone(baseArgs) + if len(testFiles) > 0 { + args = append(args, "--runTestsByPath") + args = append(args, testFiles...) + } + return command, args +} + func (j *Jest) discoveryEnv() map[string]string { envMap := make(map[string]string, len(j.platformEnv)+1) maps.Copy(envMap, j.platformEnv) diff --git a/internal/framework/jest_test.go b/internal/framework/jest_test.go index fe15bd1b..fa3f1d5e 100644 --- a/internal/framework/jest_test.go +++ b/internal/framework/jest_test.go @@ -501,3 +501,17 @@ func TestJest_RunTests_WithOverride(t *testing.T) { t.Errorf("expected args %v, got %v", expectedArgs, capturedArgs) } } + +func TestJest_TestCommandRunsFullSuiteWithoutFileSelection(t *testing.T) { + jest := &Jest{commandOverride: []string{"npm", "test", "--", "--runInBand"}} + + command, args := jest.TestCommand(nil) + + if command != "npm" { + t.Errorf("expected command %q, got %q", "npm", command) + } + expectedArgs := []string{"test", "--", "--runInBand"} + if !slices.Equal(args, expectedArgs) { + t.Errorf("expected args %v, got %v", expectedArgs, args) + } +} diff --git a/internal/framework/minitest.go b/internal/framework/minitest.go index 4a94ed9f..21219e26 100644 --- a/internal/framework/minitest.go +++ b/internal/framework/minitest.go @@ -47,6 +47,14 @@ func (m *Minitest) Name() string { return "minitest" } +func (m *Minitest) Detect(repositoryRoot string) (bool, error) { + detected, err := detectAnyPath(repositoryRoot, minitestRootDir) + if err != nil || detected { + return detected, err + } + return detectFileContaining(repositoryRoot, "Gemfile", "minitest") +} + func (m *Minitest) DiscoverTests(ctx context.Context, testFiles discovery.TestFileSet) ([]testoptimization.Test, error) { discovery.Cleanup() diff --git a/internal/framework/mocha.go b/internal/framework/mocha.go index 7c697a74..281121e1 100644 --- a/internal/framework/mocha.go +++ b/internal/framework/mocha.go @@ -46,7 +46,10 @@ func NewMocha() *Mocha { func (m *Mocha) SetPlatformEnv(platformEnv map[string]string) { m.platformEnv = platformEnv } func (m *Mocha) GetPlatformEnv() map[string]string { return m.platformEnv } func (m *Mocha) Name() string { return "mocha" } -func (m *Mocha) SupportsFullTestDiscovery() bool { return false } +func (m *Mocha) Detect(repositoryRoot string) (bool, error) { + return detectJavaScriptFramework(repositoryRoot, "mocha") +} +func (m *Mocha) SupportsFullTestDiscovery() bool { return false } func (m *Mocha) SourceFileForSuite(suite string) (string, bool) { suite = strings.TrimSpace(suite) diff --git a/internal/framework/playwright.go b/internal/framework/playwright.go index 2ef99b28..9b42459c 100644 --- a/internal/framework/playwright.go +++ b/internal/framework/playwright.go @@ -59,7 +59,10 @@ func NewPlaywright() *Playwright { func (p *Playwright) SetPlatformEnv(platformEnv map[string]string) { p.platformEnv = platformEnv } func (p *Playwright) GetPlatformEnv() map[string]string { return p.platformEnv } func (p *Playwright) Name() string { return "playwright" } -func (p *Playwright) SupportsFullTestDiscovery() bool { return false } +func (p *Playwright) Detect(repositoryRoot string) (bool, error) { + return detectJavaScriptFramework(repositoryRoot, "@playwright/test", "playwright") +} +func (p *Playwright) SupportsFullTestDiscovery() bool { return false } func (p *Playwright) SourceFileForSuite(suite string) (string, bool) { suite = strings.TrimSpace(suite) diff --git a/internal/framework/pytest.go b/internal/framework/pytest.go index 0eee3765..c14ed7c5 100644 --- a/internal/framework/pytest.go +++ b/internal/framework/pytest.go @@ -44,6 +44,20 @@ func (p *PyTest) Name() string { return "pytest" } +func (p *PyTest) Detect(repositoryRoot string) (bool, error) { + detected, err := detectAnyPath(repositoryRoot, "pytest.ini", "conftest.py") + if err != nil || detected { + return detected, err + } + for _, filename := range []string{"pyproject.toml", "requirements.txt", "setup.py", "setup.cfg", "tox.ini"} { + detected, err = detectFileContaining(repositoryRoot, filename, "pytest") + if err != nil || detected { + return detected, err + } + } + return false, nil +} + // TestPattern returns the glob pattern used to discover pytest test files. // Priority: explicit --tests-location flag > pytest config file > built-in default. // Multiple testpaths or python_files from config are collapsed into brace-expansion diff --git a/internal/framework/rspec.go b/internal/framework/rspec.go index 61e16880..ee0d39e5 100644 --- a/internal/framework/rspec.go +++ b/internal/framework/rspec.go @@ -49,6 +49,14 @@ func (r *RSpec) Name() string { return "rspec" } +func (r *RSpec) Detect(repositoryRoot string) (bool, error) { + detected, err := detectAnyPath(repositoryRoot, ".rspec") + if err != nil || detected { + return detected, err + } + return detectFileContaining(repositoryRoot, "Gemfile", "rspec") +} + func (r *RSpec) DiscoverTests(ctx context.Context, testFiles discovery.TestFileSet) ([]testoptimization.Test, error) { discovery.Cleanup() diff --git a/internal/framework/testdrive.go b/internal/framework/testdrive.go new file mode 100644 index 00000000..b9b814ad --- /dev/null +++ b/internal/framework/testdrive.go @@ -0,0 +1,113 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package framework + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + + "github.com/DataDog/ddtest/internal/settings" + "github.com/kballard/go-shellquote" +) + +// TestdriveCommand uses a project's package script when available, preserving +// its setup and runner configuration. Otherwise it uses the existing runner. +// Detection and preview must not launch subprocesses. +func TestdriveCommand(root string, runner Framework) (string, []string, error) { + if command := strings.TrimSpace(settings.GetCommand()); command != "" { + override, err := shellquote.Split(command) + if err != nil { + return "", nil, fmt.Errorf("parse testdrive --command: %w", err) + } + if len(override) == 0 { + return "", nil, fmt.Errorf("testdrive --command is empty") + } + return override[0], override[1:], nil + } + manifest, found, err := readPackageManifest(root) + if err != nil { + return "", nil, err + } + if found { + var scripts []string + for name, script := range manifest.Scripts { + if strings.Contains(script, runner.Name()) { + scripts = append(scripts, name) + } + } + slices.Sort(scripts) + if slices.Contains(scripts, "test") { + scripts = []string{"test"} + } + if len(scripts) == 1 { + manager := "npm" + for _, lock := range []struct{ file, manager string }{{"pnpm-lock.yaml", "pnpm"}, {"yarn.lock", "yarn"}, {"bun.lock", "bun"}, {"bun.lockb", "bun"}} { + if _, err := os.Stat(filepath.Join(root, lock.file)); err == nil { + manager = lock.manager + break + } + } + args := []string{"run", scripts[0]} + if manager == "npm" && scripts[0] == "test" { + args = []string{"test"} + } + extra := []string{} + if runner.Name() == "jest" { + extra = append(extra, "--runInBand") + } + if runner.Name() == "vitest" { + extra = append(extra, "--run") + } + if len(extra) > 0 && manager == "npm" { + args = append(args, "--") + } + return manager, append(args, extra...), nil + } + if len(scripts) > 1 { + return "", nil, fmt.Errorf("multiple %s scripts (%s); choose the test entry point with --command", runner.Name(), strings.Join(scripts, ", ")) + } + } + switch f := runner.(type) { + case *Jest: + command, args := f.getJestCommand() + return command, args, nil + case *Mocha: + command, args := f.getMochaCommand() + return command, args, nil + case *Cucumber: + command, args := f.getCucumberCommand() + return command, args, nil + case *Vitest: + command, args := f.getVitestCommand() + return command, vitestArgsForSubcommand(args, "run"), nil + case *Playwright: + command, args := f.getPlaywrightCommand() + return command, playwrightRunArgs(command, args, nil), nil + case *Cypress: + command, args := f.getCypressCommand() + return command, cypressRunArgs(command, args, nil), nil + case *PyTest: + interpreter := "python" + if _, err := exec.LookPath(interpreter); err != nil { + interpreter = "python3" + } + return interpreter, []string{"-m", "pytest"}, nil + case *RSpec: + command, args := f.getRSpecCommand() + return command, args, nil + case *Minitest: + if _, err := os.Stat(filepath.Join(root, "bin", "rails")); err == nil { + return "bin/rails", []string{"test"}, nil + } + return "bundle", []string{"exec", "rake", "test"}, nil + default: + return "", nil, fmt.Errorf("unsupported testdrive framework: %s", runner.Name()) + } +} diff --git a/internal/framework/testdrive_test.go b/internal/framework/testdrive_test.go new file mode 100644 index 00000000..b533bab5 --- /dev/null +++ b/internal/framework/testdrive_test.go @@ -0,0 +1,76 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package framework + +import ( + "os" + "path/filepath" + "testing" + + "github.com/DataDog/ddtest/internal/settings" + "github.com/stretchr/testify/require" +) + +func TestTestdriveUsesPackageManagerAndProjectScript(t *testing.T) { + settings.Get().Command = "" + t.Cleanup(func() { settings.Get().Command = "" }) + for _, manager := range []struct { + lock, command string + args []string + }{ + {"", "npm", []string{"run", "test:unit", "--", "--run"}}, + {"yarn.lock", "yarn", []string{"run", "test:unit", "--run"}}, + {"pnpm-lock.yaml", "pnpm", []string{"run", "test:unit", "--run"}}, + {"bun.lock", "bun", []string{"run", "test:unit", "--run"}}, + } { + t.Run(manager.command, func(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"scripts":{"test:unit":"vitest --config custom.ts"}}`), 0644)) + if manager.lock != "" { + require.NoError(t, os.WriteFile(filepath.Join(root, manager.lock), nil, 0644)) + } + command, args, err := TestdriveCommand(root, NewVitest()) + require.NoError(t, err) + require.Equal(t, manager.command, command) + require.Equal(t, manager.args, args) + }) + } +} + +func TestTestdrivePreservesExplicitCommandArguments(t *testing.T) { + t.Cleanup(func() { settings.Get().Command = "" }) + settings.Get().Command = `npm run smoke -- --config "config with spaces.js"` + command, args, err := TestdriveCommand(t.TempDir(), NewMocha()) + require.NoError(t, err) + require.Equal(t, "npm", command) + require.Equal(t, []string{"run", "smoke", "--", "--config", "config with spaces.js"}, args) + settings.Get().Command = `npm "` + _, _, err = TestdriveCommand(t.TempDir(), NewMocha()) + require.ErrorContains(t, err, "parse testdrive") +} + +func TestTestdriveDoesNotGuessBetweenScripts(t *testing.T) { + settings.Get().Command = "" + t.Cleanup(func() { settings.Get().Command = "" }) + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"scripts":{"unit":"mocha test/unit","integration":"mocha test/integration"}}`), 0644)) + _, _, err := TestdriveCommand(root, NewMocha()) + require.ErrorContains(t, err, "--command") +} + +func TestJestDetectionSupportsNamedTestScripts(t *testing.T) { + settings.Get().Command = "" + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "package.json"), []byte(`{"scripts":{"test:unit":"jest"},"devDependencies":{"jest":"30.5.1"}}`), 0644)) + jest := NewJest() + found, err := jest.Detect(root) + require.NoError(t, err) + require.True(t, found) + command, args, err := TestdriveCommand(root, jest) + require.NoError(t, err) + require.Equal(t, "npm", command) + require.Equal(t, []string{"run", "test:unit", "--", "--runInBand"}, args) +} diff --git a/internal/framework/vitest.go b/internal/framework/vitest.go index 8a306204..63d5b08a 100644 --- a/internal/framework/vitest.go +++ b/internal/framework/vitest.go @@ -62,6 +62,10 @@ func (v *Vitest) Name() string { return "vitest" } +func (v *Vitest) Detect(repositoryRoot string) (bool, error) { + return detectJavaScriptFramework(repositoryRoot, "vitest") +} + // Vitest is planned and skipped at suite (test file) level. Native full test // discovery is unnecessary for that mode. func (v *Vitest) SupportsFullTestDiscovery() bool { diff --git a/internal/onboard/instructions/github.md b/internal/onboard/instructions/github.md new file mode 100644 index 00000000..e75b8527 --- /dev/null +++ b/internal/onboard/instructions/github.md @@ -0,0 +1,43 @@ +# Enable Datadog Test Optimization for __FRAMEWORK__ on GitHub Actions + +Apply this setup to every GitHub Actions job that runs __FRAMEWORK__. + +## 1. Instrument the test job + +Add this step after checkout and dependency installation, immediately before the first test step: + +```yaml +- name: Configure Datadog Test Optimization + uses: datadog/test-visibility-github-action@v3 + with: + languages: __LANGUAGE__ + __TRACER_SETTING__ + api_key: ${{ secrets.DD_API_KEY }} + site: datadoghq.com +``` + +If the organization uses a Datadog site other than US1, replace `datadoghq.com` with that site. + +__BOOTSTRAP__ + +Keep the existing test command and unrelated workflow content unchanged. Add the Datadog action once per test job, not once per test step. + +## 2. Try it locally + +Run the local, credential-free setup check: + +```shell +ddtest testdrive +``` + +After it finishes, post every `Open report:` link printed by `ddtest testdrive` to the user so they can open the local Test Optimization report. Do not only summarize the result. + +## 3. Ask a human to connect Datadog + +The API key must be created and added to GitHub by a human. Ask the human to: + +1. Create a [Datadog API key](https://app.datadoghq.com/organization-settings/api-keys). +2. Add it to the GitHub repository as a secret named `DD_API_KEY`. +3. Tell you when the secret is ready without sharing the key itself. + +Do not ask the human to paste the API key into chat, and do not try to create or read the secret yourself. After the human confirms it is ready, commit and push the workflow change. The GitHub Actions run verifies the real Datadog backend connection. diff --git a/internal/onboard/multiframework_test.go b/internal/onboard/multiframework_test.go new file mode 100644 index 00000000..487cfa88 --- /dev/null +++ b/internal/onboard/multiframework_test.go @@ -0,0 +1,55 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOnboardAllSupportedFrameworks(t *testing.T) { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest", "rspec", "minitest"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + language := "js" + entry := name + files := map[string]string{"package.json": `{"scripts":{"test":"` + name + `"}}`} + switch name { + case "pytest": + files = map[string]string{"pyproject.toml": "[tool.pytest.ini_options]\n"} + language = "python" + entry = "uv run tox" + case "rspec", "minitest": + files = map[string]string{"Gemfile": "gem '" + name + "'\n"} + language = "ruby" + entry = "bundle exec rake" + } + for file, contents := range files { + require.NoError(t, os.WriteFile(filepath.Join(root, file), []byte(contents), 0644)) + } + require.NoError(t, os.MkdirAll(filepath.Join(root, ".github", "workflows"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".github", "workflows", "ci.yml"), []byte("jobs:\n tests:\n steps:\n - run: "+entry+"\n"), 0644)) + var output bytes.Buffer + require.NoError(t, Run(root, &output)) + require.Contains(t, output.String(), "languages: "+language) + require.Contains(t, output.String(), "ddtest testdrive --framework "+name) + require.Contains(t, output.String(), "Ask a human to connect Datadog") + if language != "js" { + require.NotContains(t, output.String(), "NODE_OPTIONS") + } + if name == "vitest" { + require.Contains(t, output.String(), "DD_TRACE_ESM_IMPORT") + } + if name == "cypress" { + require.Contains(t, output.String(), "setupNodeEvents") + } + }) + } +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go new file mode 100644 index 00000000..52e129e3 --- /dev/null +++ b/internal/onboard/onboard.go @@ -0,0 +1,163 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +// Package onboard prints the smallest supported Test Optimization setup. +package onboard + +import ( + _ "embed" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/testdrive/tracer" +) + +const githubAction = "datadog/test-visibility-github-action" + +//go:embed instructions/github.md +var gitHubInstructions string + +// Run detects the first supported onboarding path and prints its instructions. +func Run(repositoryRoot string, output io.Writer) error { + return RunWithFramework(repositoryRoot, "", output) +} + +func RunWithFramework(repositoryRoot, hint string, output io.Writer) error { + language, runner, err := platform.DetectTestProject(repositoryRoot, hint) + if err != nil { + return err + } + name := runner.Name() + + workflows, configured, err := findWorkflows(repositoryRoot, language, name) + if err != nil { + return err + } + if len(workflows) == 0 { + return fmt.Errorf("onboard could not find a GitHub Actions workflow that runs %s", name) + } + + _, _ = fmt.Fprintf(output, "DDTest found %s, %s, and GitHub Actions.\n", language, name) + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, "Test workflow(s):") + for _, workflow := range workflows { + _, _ = fmt.Fprintf(output, " - %s\n", workflow) + } + + if len(configured) == len(workflows) { + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, "Datadog Test Optimization already appears in every detected test workflow.") + _, _ = fmt.Fprintf(output, "Run `ddtest testdrive --framework %s` to check the setup locally.\n", name) + _, _ = fmt.Fprintln(output, "After it finishes, post every `Open report:` link to the user so they can open the local Test Optimization report.") + return nil + } + + _, _ = fmt.Fprintln(output) + _, _ = fmt.Fprintln(output, strings.TrimSpace(instructions(language, name))) + return nil +} + +func findWorkflows(repositoryRoot, language, name string) ([]string, []string, error) { + patterns := []string{ + filepath.Join(repositoryRoot, ".github", "workflows", "*.yml"), + filepath.Join(repositoryRoot, ".github", "workflows", "*.yaml"), + } + + var workflows []string + var configured []string + for _, pattern := range patterns { + matches, err := filepath.Glob(pattern) + if err != nil { + return nil, nil, fmt.Errorf("find GitHub Actions workflows: %w", err) + } + for _, path := range matches { + contents, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read %s: %w", path, err) + } + text := strings.ToLower(string(contents)) + if !looksLikeTestWorkflow(text, language, name) { + continue + } + + relativePath, err := filepath.Rel(repositoryRoot, path) + if err != nil { + return nil, nil, fmt.Errorf("make workflow path relative: %w", err) + } + relativePath = filepath.ToSlash(relativePath) + workflows = append(workflows, relativePath) + if strings.Contains(text, githubAction) { + configured = append(configured, relativePath) + } + } + } + + sort.Strings(workflows) + sort.Strings(configured) + return workflows, configured, nil +} + +func looksLikeTestWorkflow(workflow, language, name string) bool { + markers := []string{name, githubAction} + switch language { + case "javascript": + markers = append(markers, "npm test", "npm run test", "yarn test", "yarn run test", "pnpm test", "pnpm run test", "bun test", "bun run test") + case "ruby": + markers = append(markers, "bundle exec rake", "rake test", "rails test") + case "python": + markers = append(markers, "tox", "nox") + } + for _, marker := range markers { + if strings.Contains(workflow, marker) { + return true + } + } + return false +} + +func instructions(language, name string) string { + actionLanguage := language + var bootstrap, tracerSetting string + switch language { + case "javascript": + actionLanguage = "js" + tracerSetting = "js-tracer-version: " + tracer.JavaScriptVersion + bootstrap = javascriptBootstrap + if name == "cypress" { + bootstrap += "\n\n" + cypressBootstrap + } + if name == "cucumber" { + bootstrap += "\n\nFor the pinned tracer, also set DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED=false on the test step. This avoids a tracer crash on Cucumber Background/Rule nodes; basic reporting is unaffected." + } + case "python": + bootstrap = pythonBootstrap + tracerSetting = "python-tracer-version: " + tracer.PythonVersion + case "ruby": + bootstrap = rubyBootstrap + tracerSetting = "ruby-tracer-version: " + tracer.RubyVersion + } + text := strings.NewReplacer("__FRAMEWORK__", name, "__LANGUAGE__", actionLanguage, "__BOOTSTRAP__", bootstrap, "__TRACER_SETTING__", tracerSetting).Replace(gitHubInstructions) + return strings.ReplaceAll(text, "ddtest testdrive", "ddtest testdrive --framework "+name) +} + +const javascriptBootstrap = "Use Node.js 22 or newer. GitHub Actions cannot set NODE_OPTIONS for later steps, so merge this into the existing test step, preserving any current Node options:\n\n```yaml\nenv:\n NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }} --import ${{ env.DD_TRACE_ESM_IMPORT }}\n```\n\nThe --import loader is required for Vitest and other ESM tests." + +const pythonBootstrap = "The action exports PYTHONPATH and PYTEST_ADDOPTS=--ddtrace for pytest. Preserve these variables on the existing test step; do not replace its current arguments. Activate the same Python environment used for the tests before the action. If CI uses tox or nox, pass DD_*, PYTHONPATH, and PYTEST_ADDOPTS into the test environment." + +const rubyBootstrap = "The action installs datadog-ci into the CI bundle and exports RUBYOPT=-rbundler/setup -rdatadog/ci/auto_instrument. Preserve RUBYOPT on the RSpec or Minitest step and merge any existing Ruby options. Keep the existing bundle exec or binstub entry point." + +const cypressBootstrap = `Cypress also needs browser-side instrumentation; NODE_OPTIONS alone is insufficient: + +1. In the existing Cypress config, resolve the tracer root from the action's DD_TRACE_PACKAGE value: path.dirname(path.dirname(process.env.DD_TRACE_PACKAGE)). Only enable this configuration when that variable is present. +2. Compose the existing setupNodeEvents callback with require(path.join(tracerRoot, 'ci/cypress/plugin')). Preserve existing event handlers, including after:run and after:spec; do not replace them. +3. In setupNodeEvents, generate a support wrapper under RUNNER_TEMP. Write a literal require of path.join(tracerRoot, 'ci/cypress/support'), followed by a literal require of the existing resolved supportFile (unless it is false). Set the returned config.supportFile to that wrapper. Generating literal absolute imports lets Cypress's browser bundler resolve the isolated tracer without adding it to package.json. +4. Keep the project's existing application startup, browser installation, and cypress run command. Do not use cypress open. + +Use the same pattern for e2e or component configuration, whichever the selected job runs. The local testdrive creates equivalent wrappers inside its session automatically. See https://docs.datadoghq.com/tests/setup/javascript/ for the Cypress plugin contract.` diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go new file mode 100644 index 00000000..1584459d --- /dev/null +++ b/internal/onboard/onboard_test.go @@ -0,0 +1,102 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package onboard + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunPrintsJestGitHubInstructions(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + test: + steps: + - run: npm test +`) + + var output bytes.Buffer + if err := Run(repositoryRoot, &output); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + for _, expected := range []string{ + "DDTest found javascript, jest, and GitHub Actions.", + ".github/workflows/test.yml", + "datadog/test-visibility-github-action@v3", + "api_key: ${{ secrets.DD_API_KEY }}", + "NODE_OPTIONS: -r ${{ env.DD_TRACE_PACKAGE }}", + "ddtest testdrive", + "post every `Open report:` link", + "Ask a human to connect Datadog", + "without sharing the key itself", + } { + if !strings.Contains(output.String(), expected) { + t.Errorf("Run() output does not contain %q:\n%s", expected, output.String()) + } + } + if strings.Index(output.String(), "ddtest testdrive") > strings.Index(output.String(), "Ask a human to connect Datadog") { + t.Fatalf("API key setup must come after the local testdrive:\n%s", output.String()) + } +} + +func TestRunRecognizesExistingGitHubAction(t *testing.T) { + repositoryRoot := newJestRepository(t, ` +name: tests +jobs: + test: + steps: + - uses: datadog/test-visibility-github-action@v3 + - run: npm test +`) + + var output bytes.Buffer + if err := Run(repositoryRoot, &output); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + if !strings.Contains(output.String(), "already appears in every detected test workflow") { + t.Fatalf("Run() did not recognize the existing setup:\n%s", output.String()) + } + if !strings.Contains(output.String(), "ddtest testdrive") { + t.Fatalf("Run() did not print the next step:\n%s", output.String()) + } + if !strings.Contains(output.String(), "post every `Open report:` link") { + t.Fatalf("Run() did not tell the agent to share the report:\n%s", output.String()) + } +} + +func TestRunRequiresGitHubJestWorkflow(t *testing.T) { + repositoryRoot := newJestRepository(t, "name: lint\njobs:\n lint:\n steps:\n - run: npm run lint\n") + + err := Run(repositoryRoot, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "could not find a GitHub Actions workflow that runs jest") { + t.Fatalf("Run() error = %v", err) + } +} + +func newJestRepository(t *testing.T, workflow string) string { + t.Helper() + repositoryRoot := t.TempDir() + if err := os.WriteFile(filepath.Join(repositoryRoot, "package.json"), []byte(`{ + "scripts": {"test": "jest"}, + "devDependencies": {"jest": "30.0.0"} +}`), 0644); err != nil { + t.Fatal(err) + } + workflowDirectory := filepath.Join(repositoryRoot, ".github", "workflows") + if err := os.MkdirAll(workflowDirectory, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workflowDirectory, "test.yml"), []byte(workflow), 0644); err != nil { + t.Fatal(err) + } + return repositoryRoot +} diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 0a6ce776..d7aa59c0 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -101,6 +101,10 @@ func (m *MockPlatform) Name() string { return m.PlatformName } +func (m *MockPlatform) Detect(string) (bool, error) { + return true, nil +} + func (m *MockPlatform) CreateTagsMap(context.Context) (map[string]string, error) { return m.Tags, m.TagsErr } @@ -211,6 +215,10 @@ func (m *MockFramework) Name() string { return m.FrameworkName } +func (m *MockFramework) Detect(string) (bool, error) { + return true, nil +} + func (m *MockFramework) TestPattern() string { if m.TestPatternValue != "" { return m.TestPatternValue diff --git a/internal/platform/javascript.go b/internal/platform/javascript.go index b7ffaf8a..1670b8fb 100644 --- a/internal/platform/javascript.go +++ b/internal/platform/javascript.go @@ -41,6 +41,10 @@ func (j *JavaScript) Name() string { return "javascript" } +func (j *JavaScript) Detect(repositoryRoot string) (bool, error) { + return detectAnyFile(repositoryRoot, "package.json") +} + func (j *JavaScript) TestSkippingLevel() settings.TestSkippingLevel { return settings.TestSkippingLevelSuite } diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 8bcf50fb..357000a7 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -3,6 +3,8 @@ package platform import ( "context" "fmt" + "os" + "path/filepath" "strings" "github.com/DataDog/ddtest/internal/framework" @@ -11,12 +13,30 @@ import ( type Platform interface { Name() string + Detect(repositoryRoot string) (bool, error) CreateTagsMap(ctx context.Context) (map[string]string, error) DetectFramework() (framework.Framework, error) SanityCheck(ctx context.Context) error TestSkippingLevel() settings.TestSkippingLevel } +func detectAnyFile(repositoryRoot string, filenames ...string) (bool, error) { + for _, filename := range filenames { + path := filepath.Join(repositoryRoot, filename) + info, err := os.Stat(path) + if err == nil { + if !info.IsDir() { + return true, nil + } + continue + } + if !os.IsNotExist(err) { + return false, fmt.Errorf("inspect %s: %w", path, err) + } + } + return false, nil +} + // PlatformDetector defines interface for detecting platforms - needed to allow mocking in tests type PlatformDetector interface { DetectPlatform(ctx context.Context) (Platform, error) diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go index ef5592bc..946b464e 100644 --- a/internal/platform/platform_test.go +++ b/internal/platform/platform_test.go @@ -18,6 +18,44 @@ func TestNewPlatformDetector(t *testing.T) { } } +func TestPlatformsDetectTheirProjectFiles(t *testing.T) { + tests := []struct { + name string + platform Platform + filename string + }{ + {name: "ruby", platform: NewRuby(settings.TestSkippingLevelSuite), filename: "Gemfile"}, + {name: "javascript", platform: NewJavaScript(), filename: "package.json"}, + {name: "python pyproject", platform: NewPython(), filename: "pyproject.toml"}, + {name: "python setup", platform: NewPython(), filename: "setup.py"}, + {name: "python requirements", platform: NewPython(), filename: "requirements.txt"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repositoryRoot := t.TempDir() + detected, err := test.platform.Detect(repositoryRoot) + if err != nil { + t.Fatalf("Detect() unexpected error: %v", err) + } + if detected { + t.Fatal("Detect() = true for an empty repository") + } + + if err := os.WriteFile(filepath.Join(repositoryRoot, test.filename), []byte("{}"), 0644); err != nil { + t.Fatal(err) + } + detected, err = test.platform.Detect(repositoryRoot) + if err != nil { + t.Fatalf("Detect() unexpected error: %v", err) + } + if !detected { + t.Fatal("Detect() = false for a matching repository") + } + }) + } +} + func TestPlatformSanityChecksPropagateContext(t *testing.T) { type contextKey struct{} ctx := context.WithValue(context.Background(), contextKey{}, "sanity-check") diff --git a/internal/platform/python.go b/internal/platform/python.go index 1e350738..fe247d5b 100644 --- a/internal/platform/python.go +++ b/internal/platform/python.go @@ -52,6 +52,10 @@ func (p *Python) Name() string { return "python" } +func (p *Python) Detect(repositoryRoot string) (bool, error) { + return detectAnyFile(repositoryRoot, "pyproject.toml", "setup.py", "setup.cfg", "requirements.txt", "pytest.ini") +} + func (p *Python) TestSkippingLevel() settings.TestSkippingLevel { return settings.TestSkippingLevelTest } diff --git a/internal/platform/ruby.go b/internal/platform/ruby.go index f28a870a..baef6e4f 100644 --- a/internal/platform/ruby.go +++ b/internal/platform/ruby.go @@ -44,6 +44,10 @@ func (r *Ruby) Name() string { return "ruby" } +func (r *Ruby) Detect(repositoryRoot string) (bool, error) { + return detectAnyFile(repositoryRoot, "Gemfile") +} + func (r *Ruby) TestSkippingLevel() settings.TestSkippingLevel { return r.testSkippingLevel } diff --git a/internal/platform/testproject.go b/internal/platform/testproject.go new file mode 100644 index 00000000..b3e57008 --- /dev/null +++ b/internal/platform/testproject.go @@ -0,0 +1,60 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package platform + +import ( + "fmt" + "strings" + + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/settings" +) + +// DetectTestProject performs read-only detection for onboarding and testdrive. +// A framework hint resolves repositories containing more than one test runner. +func DetectTestProject(root, hint string) (string, framework.Framework, error) { + candidates := []struct { + platform Platform + frameworks []framework.Framework + }{ + {NewJavaScript(), []framework.Framework{framework.NewJest(), framework.NewMocha(), framework.NewCypress(), framework.NewPlaywright(), framework.NewCucumber(), framework.NewVitest()}}, + {NewPython(), []framework.Framework{framework.NewPytest()}}, + {NewRuby(settings.TestSkippingLevelTest), []framework.Framework{framework.NewRSpec(), framework.NewMinitest()}}, + } + var names []string + var selected framework.Framework + var language string + for _, candidate := range candidates { + found, err := candidate.platform.Detect(root) + if err != nil { + return "", nil, err + } + if !found { + continue + } + for _, runner := range candidate.frameworks { + if hint != "" && runner.Name() != hint { + continue + } + found, err := runner.Detect(root) + if err != nil { + return "", nil, err + } + if found { + names = append(names, runner.Name()) + selected = runner + language = candidate.platform.Name() + } + } + } + if len(names) == 0 { + return "", nil, fmt.Errorf("could not detect a supported test framework %q; inspect package.json, Python test configuration, or Gemfile", hint) + } + if len(names) > 1 { + return "", nil, fmt.Errorf("found multiple test frameworks (%s); select one with --framework", strings.Join(names, ", ")) + } + return language, selected, nil +} diff --git a/internal/runner/test_helpers_test.go b/internal/runner/test_helpers_test.go index a2de1131..6b8b68a7 100644 --- a/internal/runner/test_helpers_test.go +++ b/internal/runner/test_helpers_test.go @@ -51,6 +51,10 @@ func (m *MockPlatform) Name() string { return m.PlatformName } +func (m *MockPlatform) Detect(string) (bool, error) { + return true, nil +} + func (m *MockPlatform) CreateTagsMap(context.Context) (map[string]string, error) { return m.Tags, m.TagsErr } @@ -93,6 +97,10 @@ func (m *MockFramework) Name() string { return m.FrameworkName } +func (m *MockFramework) Detect(string) (bool, error) { + return true, nil +} + func (m *MockFramework) TestPattern() string { return m.TestPatternValue } diff --git a/internal/testdrive/cypress.go b/internal/testdrive/cypress.go new file mode 100644 index 00000000..ed210d09 --- /dev/null +++ b/internal/testdrive/cypress.go @@ -0,0 +1,104 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// prepareCypress wraps configuration inside the session; project config and +// support files are neither edited nor replaced on disk. +func prepareCypress(root, directory, preload string, args []string) ([]string, error) { + var config string + clean := make([]string, 0, len(args)+2) + for i := 0; i < len(args); i++ { + if args[i] == "--config-file" { + if i+1 == len(args) { + return nil, fmt.Errorf("--config-file requires a path") + } + i++ + config = args[i] + continue + } + if strings.HasPrefix(args[i], "--config-file=") { + config = strings.TrimPrefix(args[i], "--config-file=") + continue + } + clean = append(clean, args[i]) + } + if config == "" { + for _, name := range []string{"cypress.config.ts", "cypress.config.js", "cypress.config.mjs", "cypress.config.cjs"} { + if _, err := os.Stat(filepath.Join(root, name)); err == nil { + config = name + break + } + } + } + if config == "" { + return nil, fmt.Errorf("cypress needs a config file; provide its path with --command '... --config-file path'") + } + if !filepath.IsAbs(config) { + config = filepath.Join(root, config) + } + values, _ := json.Marshal(map[string]string{"root": root, "config": config, "directory": directory, "tracer": filepath.Dir(filepath.Dir(preload))}) + wrapper := "const options = " + string(values) + ";\n" + cypressWrapper + path := filepath.Join(directory, "cypress.config.cjs") + if err := os.WriteFile(path, []byte(wrapper), 0600); err != nil { + return nil, err + } + return append(clean, "--config-file", path), nil +} + +const cypressWrapper = ` +const fs = require('node:fs'); +const path = require('node:path'); +const { pathToFileURL } = require('node:url'); +// Cypress starts the config process in the wrapper's directory. Keep relative +// filesystem operations in the customer's config and hooks rooted in the project. +process.chdir(options.root); +module.exports = (async () => { + let original; + try { original = require(options.config); } + catch (error) { + if (error.code !== 'ERR_REQUIRE_ESM') throw error; + original = await import(pathToFileURL(options.config).href); + } + original = original.default || original; + const config = { ...original }; + for (const type of ['e2e', 'component']) { + if (!original[type]) continue; + const setup = original[type].setupNodeEvents; + config[type] = { ...original[type], async setupNodeEvents(on, resolved) { + // Compose hooks so adding instrumentation cannot discard customer hooks. + const handlers = new Map(); + const collect = (event, handler) => { + if (event === 'task') { handlers.set(event, { ...(handlers.get(event) || {}), ...handler }); return; } + const previous = handlers.get(event); + handlers.set(event, previous ? async (...args) => { + const first = await previous(...args); + const second = await handler(...args); + return second === undefined ? first : second; + } : handler); + }; + if (setup) resolved = (await setup(collect, resolved)) || resolved; + const support = path.join(options.directory, type + '-support.cjs'); + let contents = 'require(' + JSON.stringify(path.join(options.tracer, 'ci/cypress/support')) + ');\n'; + if (resolved.supportFile) contents += 'require(' + JSON.stringify(path.resolve(resolved.projectRoot, resolved.supportFile)) + ');\n'; + fs.writeFileSync(support, contents); + resolved.supportFile = support; + const instrument = require(path.join(options.tracer, 'ci/cypress/plugin')); + resolved = (await instrument(collect, resolved)) || resolved; + for (const [event, handler] of handlers) on(event, handler); + return resolved; + }}; + } + return config; +})(); +` diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go new file mode 100644 index 00000000..88d057ad --- /dev/null +++ b/internal/testdrive/frameworks_integration_test.go @@ -0,0 +1,129 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestPublicFrameworkTestdrives exercises the shipped CLI, real tracers, and +// real test runners. Browser downloads are explicit test setup, never testdrive +// side effects. Run with DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1. +func TestPublicFrameworkTestdrives(t *testing.T) { + if os.Getenv("DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST") == "" { + t.Skip("set DDTEST_RUN_FRAMEWORK_INTEGRATION_TEST=1") + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) + defer cancel() + binary := filepath.Join(t.TempDir(), "ddtest") + integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + fixtures := []struct { + name, manifest, command string + files map[string]string + }{ + {"jest", `{"scripts":{"test":"jest"},"devDependencies":{"jest":"30.5.1"}}`, "npm test", map[string]string{"one.test.js": `test('adds', () => expect(1+1).toBe(2));`}}, + {"mocha", `{"scripts":{"test":"mocha"},"devDependencies":{"mocha":"11.7.5"}}`, "npm test", map[string]string{"test/one.js": `const assert = require('node:assert'); it('adds', () => assert.equal(1+1,2));`}}, + {"vitest", `{"type":"module","scripts":{"test":"vitest run"},"devDependencies":{"vitest":"3.2.4"}}`, "npm test", map[string]string{"one.test.js": `import {test,expect} from 'vitest'; test('adds', () => expect(1+1).toBe(2));`}}, + {"playwright", `{"scripts":{"test":"playwright test"},"devDependencies":{"@playwright/test":"1.55.1"}}`, "npm test", map[string]string{"one.spec.js": `const {test,expect} = require('@playwright/test'); test('adds', () => expect(1+1).toBe(2));`}}, + {"cucumber", `{"scripts":{"test":"cucumber-js"},"devDependencies":{"@cucumber/cucumber":"12.2.0"}}`, "npm test", map[string]string{"features/one.feature": "Feature: Arithmetic\n Background:\n Given addition works\n Scenario: Add\n Given addition works\n", "features/step_definitions/one.js": `const {Given} = require('@cucumber/cucumber'); Given('addition works', () => require('node:assert').equal(1+1,2));`}}, + {"cypress", `{"scripts":{"test":"cypress run"},"devDependencies":{"cypress":"15.1.0"}}`, "npm test", map[string]string{"cypress.config.js": `module.exports={e2e:{supportFile:false,setupNodeEvents(on,config){on('task',{answer:()=>42});on('after:run',()=>{require('node:fs').writeFileSync('original-hook.txt','ran');});return config;}}};`, "cypress/e2e/one.cy.js": `it('preserves hooks', () => { cy.task('answer').should('equal',42); });`}}, + {"pytest", "", "python -m pytest", map[string]string{"requirements.txt": "pytest==8.4.2\n", "pytest.ini": "[pytest]\n", "test_one.py": "def test_adds():\n assert 1 + 1 == 2\n"}}, + {"rspec", "", "bundle exec rspec", map[string]string{"Gemfile": "source 'https://rubygems.org'\ngem 'rspec', '3.13.1'\n", "spec/one_spec.rb": "RSpec.describe 'addition' do\n it('adds') { expect(1 + 1).to eq(2) }\nend\n"}}, + {"minitest", "", "bundle exec rake test", map[string]string{"Gemfile": "source 'https://rubygems.org'\ngem 'rake', '13.3.0'\ngem 'minitest', '5.25.5'\n", "Rakefile": "require 'rake/testtask'\nRake::TestTask.new(:test) { |t| t.pattern = 'test/**/*_test.rb' }\n", "test/one_test.rb": "require 'minitest/autorun'\nclass AdditionTest < Minitest::Test\n def test_adds\n assert_equal 2, 1 + 1\n end\nend\n"}}, + } + for _, fixture := range fixtures { + t.Run(fixture.name, func(t *testing.T) { + name := "project space" + if fixture.name == "rspec" || fixture.name == "minitest" { + name = "project" + } // Ruby native extension builds do not support spaces. + root := filepath.Join(t.TempDir(), name) + require.NoError(t, os.MkdirAll(root, 0755)) + integrationCommand(t, ctx, root, nil, "git", "init", "-q") + for name, contents := range fixture.files { + integrationFile(t, root, name, contents) + } + if fixture.manifest != "" { + integrationFile(t, root, "package.json", fixture.manifest) + } + integrationFile(t, root, ".github/workflows/test.yml", "name: tests\non: push\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: "+fixture.command+"\n") + env := []string{} + if fixture.manifest != "" { + integrationCommand(t, ctx, root, env, "npm", "install", "--no-audit", "--no-fund") + } + if fixture.name == "pytest" { + venv := filepath.Join(t.TempDir(), "venv") + integrationCommand(t, ctx, root, nil, "python3", "-m", "venv", venv) + env = append(env, "PATH="+filepath.Join(venv, "bin")+string(os.PathListSeparator)+os.Getenv("PATH")) + integrationCommand(t, ctx, root, env, filepath.Join(venv, "bin", "python"), "-m", "pip", "install", "pytest==8.4.2") + } + if fixture.name == "rspec" || fixture.name == "minitest" { + integrationCommand(t, ctx, root, env, "bundle", "lock") + } + before := map[string]string{} + for _, name := range []string{"package.json", "package-lock.json", "Gemfile", "Gemfile.lock", "requirements.txt", "cypress.config.js"} { + contents, err := os.ReadFile(filepath.Join(root, name)) + if err == nil { + before[name] = string(contents) + } + } + onboard := integrationCommand(t, ctx, root, env, binary, "onboard") + require.Contains(t, onboard, "datadog/test-visibility-github-action@v3") + output := integrationCommand(t, ctx, root, env, binary, "testdrive", "--yes") + require.Contains(t, output, "Test Optimization is ready.") + require.Contains(t, output, "Open report:") + reports, err := filepath.Glob(filepath.Join(root, ".testoptimization", "testdrive", "*", "report.html")) + require.NoError(t, err) + require.Len(t, reports, 1) + contents, err := os.ReadFile(reports[0]) + require.NoError(t, err) + require.Contains(t, string(contents), "Test Optimization is ready.") + traffic, err := filepath.Glob(filepath.Join(filepath.Dir(reports[0]), "intake", "*citestcycle.json")) + require.NoError(t, err) + require.NotEmpty(t, traffic) + for name, contents := range before { + after, err := os.ReadFile(filepath.Join(root, name)) + require.NoError(t, err) + require.Equal(t, contents, string(after), name) + } + if _, existed := before["Gemfile.lock"]; !existed { + _, err := os.Stat(filepath.Join(root, "Gemfile.lock")) + require.True(t, os.IsNotExist(err), "project lockfile must not be created") + } + if fixture.name == "cypress" { + hook, err := os.ReadFile(filepath.Join(root, "original-hook.txt")) + require.NoError(t, err) + require.Equal(t, "ran", string(hook)) + } + t.Log(strings.TrimSpace(output)) + }) + } +} + +func integrationFile(t *testing.T, root, name, contents string) { + t.Helper() + path := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, []byte(contents), 0644)) +} + +func integrationCommand(t *testing.T, ctx context.Context, directory string, env []string, name string, args ...string) string { + t.Helper() + command := exec.CommandContext(ctx, name, args...) + command.Dir = directory + command.Env = append(os.Environ(), env...) + output, err := command.CombinedOutput() + require.NoError(t, err, "%s %v:\n%s", name, args, output) + return string(output) +} diff --git a/internal/testdrive/intake/coverage_test.go b/internal/testdrive/intake/coverage_test.go new file mode 100644 index 00000000..61adcb4a --- /dev/null +++ b/internal/testdrive/intake/coverage_test.go @@ -0,0 +1,146 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/textproto" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tinylib/msgp/msgp" +) + +func TestCoveredTestCountAssociatesSuiteCoverageWithEveryTestInTheSuite(t *testing.T) { + const sessionID = 10 + + events := msgp.AppendMapHeader(nil, 1) + events = msgp.AppendString(events, "events") + events = msgp.AppendArrayHeader(events, 2) + events = appendEvent(events, "test", sessionID, 20, 100) + events = appendEvent(events, "test", sessionID, 20, 200) + + server := serverWithCoverage(t, events, appendCoverage(nil, sessionID, 20, 0)) + + count, err := server.CoveredTestCount() + require.NoError(t, err) + require.Equal(t, 2, count) +} + +func TestCoveredTestCountAssociatesTestCoverageBySpan(t *testing.T) { + const sessionID = 10 + + events := msgp.AppendMapHeader(nil, 1) + events = msgp.AppendString(events, "events") + events = msgp.AppendArrayHeader(events, 2) + events = appendEvent(events, "test", sessionID, 20, 100) + events = appendEvent(events, "test", sessionID, 20, 200) + + server := serverWithCoverage(t, events, appendCoverage(nil, sessionID, 20, 200)) + + count, err := server.CoveredTestCount() + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestCoverageReferencesCountFiles(t *testing.T) { + events := msgp.AppendMapHeader(nil, 1) + events = msgp.AppendString(events, "events") + events = msgp.AppendArrayHeader(events, 1) + events = appendEvent(events, "test", 10, 20, 100) + server := serverWithCoverage(t, events, appendCoverage(nil, 10, 20, 100, "one.js", "two.js")) + + coverages, err := server.coverageReferences() + require.NoError(t, err) + require.Len(t, coverages, 1) + require.Equal(t, 2, coverages[0].fileCount) + require.Equal(t, []string{"one.js", "two.js"}, coverages[0].files) +} + +func TestFindingsListsEveryTestWithSuiteCoverage(t *testing.T) { + const sessionID = 10 + events := msgp.AppendMapHeader(nil, 1) + events = msgp.AppendString(events, "events") + events = msgp.AppendArrayHeader(events, 2) + events = appendDetailedTest(events, sessionID, 20, 100, "one", "one.test.js", "pass", time.Millisecond, false, "") + events = appendDetailedTest(events, sessionID, 20, 200, "two", "one.test.js", "pass", 2*time.Millisecond, false, "") + server := serverWithCoverage(t, events, appendCoverage(nil, sessionID, 20, 0, "src/one.js", "src/two.js")) + + findings, err := server.Findings() + require.NoError(t, err) + require.Equal(t, 2, findings.TestCount) + require.Len(t, findings.Tests, 2) + for _, test := range findings.Tests { + require.Equal(t, "suite", test.CoverageLevel) + require.Equal(t, []string{"src/one.js", "src/two.js"}, test.CoveredFiles) + } +} + +func serverWithCoverage(t *testing.T, events, coverageEntry []byte) *Server { + t.Helper() + + coverage := msgp.AppendMapHeader(nil, 2) + coverage = msgp.AppendString(coverage, "version") + coverage = msgp.AppendInt(coverage, 2) + coverage = msgp.AppendString(coverage, "coverages") + coverage = msgp.AppendArrayHeader(coverage, 1) + coverage = append(coverage, coverageEntry...) + + return &Server{requests: []RawRequest{ + {Method: http.MethodPost, Path: testCyclePath, Body: events}, + multipartCoverageRequest(t, coverage), + }} +} + +func appendCoverage(payload []byte, sessionID, suiteID, spanID uint64, files ...string) []byte { + fieldCount := uint32(3) + if spanID != 0 { + fieldCount++ + } + payload = msgp.AppendMapHeader(payload, fieldCount) + payload = msgp.AppendString(payload, "test_session_id") + payload = msgp.AppendUint64(payload, sessionID) + payload = msgp.AppendString(payload, "test_suite_id") + payload = msgp.AppendUint64(payload, suiteID) + if spanID != 0 { + payload = msgp.AppendString(payload, "span_id") + payload = msgp.AppendUint64(payload, spanID) + } + payload = msgp.AppendString(payload, "files") + payload = msgp.AppendArrayHeader(payload, uint32(len(files))) + for _, file := range files { + payload = msgp.AppendMapHeader(payload, 1) + payload = msgp.AppendString(payload, "filename") + payload = msgp.AppendString(payload, file) + } + return payload +} + +func multipartCoverageRequest(t *testing.T, coverage []byte) RawRequest { + t.Helper() + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + header := textproto.MIMEHeader{ + "Content-Disposition": {`form-data; name="coverage1"; filename="coverage1.msgpack"`}, + "Content-Type": {"application/msgpack"}, + } + part, err := writer.CreatePart(header) + require.NoError(t, err) + _, err = part.Write(coverage) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + return RawRequest{ + Method: http.MethodPost, + Path: testCoveragePath, + Header: http.Header{"Content-Type": {writer.FormDataContentType()}}, + Body: body.Bytes(), + } +} diff --git a/internal/testdrive/intake/events.go b/internal/testdrive/intake/events.go new file mode 100644 index 00000000..f2f94b98 --- /dev/null +++ b/internal/testdrive/intake/events.go @@ -0,0 +1,230 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/tinylib/msgp/msgp" +) + +const testCyclePath = "/api/v2/citestcycle" + +type testReference struct { + sessionID uint64 + suiteID uint64 + spanID uint64 + name string + suite string + sourceFile string + sourceStart int + sourceEnd int + status string + finalStatus string + duration time.Duration + isRetry bool + retryReason string + errorType string + errorMessage string + errorStack string +} + +type suiteReference struct { + sessionID uint64 + suiteID uint64 +} + +// TestEventCount returns the number of test events observed by the intake. +func (s *Server) TestEventCount() (int, error) { + tests, err := s.testReferences() + if err != nil { + return 0, err + } + return len(tests), nil +} + +func (s *Server) testReferences() ([]testReference, error) { + var tests []testReference + for requestIndex, request := range s.Requests() { + if request.Method != http.MethodPost || request.Path != testCyclePath { + continue + } + + body, err := uncompressRequestBody(request) + if err != nil { + return nil, fmt.Errorf("uncompress request %d to %s: %w", requestIndex+1, testCyclePath, err) + } + requestTests, err := readTests(body) + if err != nil { + return nil, fmt.Errorf( + "recognize test events in request %d to %s (content type %q, content encoding %q): %w", + requestIndex+1, + testCyclePath, + request.Header.Get("Content-Type"), + request.Header.Get("Content-Encoding"), + err, + ) + } + tests = append(tests, requestTests...) + } + return tests, nil +} + +func readTests(payload []byte) ([]testReference, error) { + fieldCount, rest, err := msgp.ReadMapHeaderBytes(payload) + if err != nil { + return nil, err + } + + var tests []testReference + for range fieldCount { + var field string + field, rest, err = msgp.ReadStringBytes(rest) + if err != nil { + return nil, err + } + + if field != "events" { + rest, err = msgp.Skip(rest) + if err != nil { + return nil, err + } + continue + } + + var requestTests []testReference + requestTests, rest, err = readTestsInEventArray(rest) + if err != nil { + return nil, err + } + tests = append(tests, requestTests...) + } + return tests, nil +} + +func readTestsInEventArray(payload []byte) ([]testReference, []byte, error) { + eventCount, rest, err := msgp.ReadArrayHeaderBytes(payload) + if err != nil { + return nil, nil, err + } + + var tests []testReference + for range eventCount { + var eventType string + var reference testReference + fieldCount, remaining, readErr := msgp.ReadMapHeaderBytes(rest) + if readErr != nil { + return nil, nil, readErr + } + rest = remaining + + for range fieldCount { + var field string + field, rest, readErr = msgp.ReadStringBytes(rest) + if readErr != nil { + return nil, nil, readErr + } + + switch field { + case "type": + eventType, rest, readErr = msgp.ReadStringBytes(rest) + case "content": + reference, rest, readErr = readTestReference(rest) + default: + rest, readErr = msgp.Skip(rest) + } + if readErr != nil { + return nil, nil, readErr + } + } + if eventType == "test" { + tests = append(tests, reference) + } + } + return tests, rest, nil +} + +func readTestReference(payload []byte) (testReference, []byte, error) { + content, rest, err := msgp.ReadMapStrIntfBytes(payload, nil) + if err != nil { + return testReference{}, nil, err + } + + reference := testReference{ + sessionID: unsigned(content["test_session_id"]), + suiteID: unsigned(content["test_suite_id"]), + spanID: unsigned(content["span_id"]), + duration: time.Duration(integer(content["duration"])), + } + if metadata, ok := content["meta"].(map[string]any); ok { + reference.name = text(metadata["test.name"]) + reference.suite = text(metadata["test.suite"]) + reference.sourceFile = text(metadata["test.source.file"]) + reference.status = strings.ToLower(text(metadata["test.status"])) + reference.finalStatus = strings.ToLower(text(metadata["test.final_status"])) + if reference.status == "" { + reference.status = reference.finalStatus + } + reference.isRetry = truthy(metadata["test.is_retry"]) + reference.retryReason = text(metadata["test.retry_reason"]) + reference.errorType = text(metadata["error.type"]) + reference.errorMessage = text(metadata["error.message"]) + reference.errorStack = text(metadata["error.stack"]) + } + if metrics, ok := content["metrics"].(map[string]any); ok { + reference.isRetry = reference.isRetry || truthy(metrics["test.is_retry"]) + reference.sourceStart = int(integer(metrics["test.source.start"])) + reference.sourceEnd = int(integer(metrics["test.source.end"])) + } + return reference, rest, nil +} + +func unsigned(value any) uint64 { + switch value := value.(type) { + case uint64: + return value + case int64: + if value > 0 { + return uint64(value) + } + } + return 0 +} + +func integer(value any) int64 { + switch value := value.(type) { + case int64: + return value + case uint64: + return int64(value) + case float64: + return int64(value) + case float32: + return int64(value) + } + return 0 +} + +func text(value any) string { + valueText, _ := value.(string) + return valueText +} + +func truthy(value any) bool { + switch value := value.(type) { + case bool: + return value + case string: + parsed, err := strconv.ParseBool(value) + return err == nil && parsed + default: + return integer(value) != 0 + } +} diff --git a/internal/testdrive/intake/events_test.go b/internal/testdrive/intake/events_test.go new file mode 100644 index 00000000..4d464da2 --- /dev/null +++ b/internal/testdrive/intake/events_test.go @@ -0,0 +1,201 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "net/http" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tinylib/msgp/msgp" +) + +func TestTestEventCountRecognizesOneTest(t *testing.T) { + payload := msgp.AppendMapHeader(nil, 2) + payload = msgp.AppendString(payload, "version") + payload = msgp.AppendInt(payload, 1) + payload = msgp.AppendString(payload, "events") + payload = msgp.AppendArrayHeader(payload, 3) + payload = appendEvent(payload, "test_session_end", 10, 20, 30) + payload = appendEvent(payload, "test", 10, 20, 30) + payload = appendEvent(payload, "test_suite_end", 10, 20, 30) + + server := &Server{requests: []RawRequest{ + {Method: http.MethodPost, Path: "/another-endpoint", Body: []byte("not msgpack")}, + {Method: http.MethodPost, Path: testCyclePath, Body: payload}, + }} + + count, err := server.TestEventCount() + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestTestReferencesReadReportFields(t *testing.T) { + payload := msgp.AppendMapHeader(nil, 1) + payload = msgp.AppendString(payload, "events") + payload = msgp.AppendArrayHeader(payload, 1) + payload = appendDetailedTest(payload, 10, 20, 30, "works", "one.test.js", "pass", 250*time.Millisecond, true, "") + + server := &Server{requests: []RawRequest{{Method: http.MethodPost, Path: testCyclePath, Body: payload}}} + tests, err := server.testReferences() + require.NoError(t, err) + require.Equal(t, []testReference{{ + sessionID: 10, + suiteID: 20, + spanID: 30, + name: "works", + suite: "one.test.js", + sourceFile: "one.test.js", + sourceStart: 12, + sourceEnd: 14, + status: "pass", + duration: 250 * time.Millisecond, + isRetry: true, + retryReason: "early_flake_detection", + }}, tests) +} + +func TestTestReferencesReadErrors(t *testing.T) { + payload := msgp.AppendMapHeader(nil, 1) + payload = msgp.AppendString(payload, "events") + payload = msgp.AppendArrayHeader(payload, 1) + payload = appendDetailedTest(payload, 10, 20, 30, "breaks", "one.test.js", "fail", time.Millisecond, false, "expected true") + + server := &Server{requests: []RawRequest{{Method: http.MethodPost, Path: testCyclePath, Body: payload}}} + tests, err := server.testReferences() + require.NoError(t, err) + require.Len(t, tests, 1) + require.Equal(t, "AssertionError", tests[0].errorType) + require.Equal(t, "expected true", tests[0].errorMessage) + require.Equal(t, "stack trace", tests[0].errorStack) +} + +func TestTestReferencesKeepAttemptAndFinalStatus(t *testing.T) { + payload := msgp.AppendMapHeader(nil, 1) + payload = msgp.AppendString(payload, "events") + payload = msgp.AppendArrayHeader(payload, 1) + payload = appendDetailedTestWithFinalStatus(payload, 10, 20, 30, "flaky", "one.test.js", "fail", "pass", time.Millisecond, true, "") + + server := &Server{requests: []RawRequest{{Method: http.MethodPost, Path: testCyclePath, Body: payload}}} + tests, err := server.testReferences() + require.NoError(t, err) + require.Len(t, tests, 1) + require.Equal(t, "fail", tests[0].status) + require.Equal(t, "pass", tests[0].finalStatus) +} + +func TestTestEventCountReportsInvalidPayload(t *testing.T) { + server := &Server{requests: []RawRequest{{ + Method: http.MethodPost, + Path: testCyclePath, + Body: []byte("not msgpack"), + }}} + + _, err := server.TestEventCount() + require.ErrorContains(t, err, "recognize test events") +} + +func appendEvent(payload []byte, eventType string, sessionID, suiteID, spanID uint64) []byte { + payload = msgp.AppendMapHeader(payload, 2) + payload = msgp.AppendString(payload, "type") + payload = msgp.AppendString(payload, eventType) + payload = msgp.AppendString(payload, "content") + payload = msgp.AppendMapHeader(payload, 3) + payload = msgp.AppendString(payload, "test_session_id") + payload = msgp.AppendUint64(payload, sessionID) + payload = msgp.AppendString(payload, "test_suite_id") + payload = msgp.AppendUint64(payload, suiteID) + payload = msgp.AppendString(payload, "span_id") + payload = msgp.AppendUint64(payload, spanID) + return payload +} + +func appendDetailedTest( + payload []byte, + sessionID, suiteID, spanID uint64, + name, suite, status string, + duration time.Duration, + isRetry bool, + errorMessage string, +) []byte { + return appendDetailedTestWithFinalStatus(payload, sessionID, suiteID, spanID, name, suite, status, "", duration, isRetry, errorMessage) +} + +func appendDetailedTestWithFinalStatus( + payload []byte, + sessionID, suiteID, spanID uint64, + name, suite, status, finalStatus string, + duration time.Duration, + isRetry bool, + errorMessage string, +) []byte { + payload = msgp.AppendMapHeader(payload, 2) + payload = msgp.AppendString(payload, "type") + payload = msgp.AppendString(payload, "test") + payload = msgp.AppendString(payload, "content") + payload = msgp.AppendMapHeader(payload, 6) + metadataFields := uint32(5) + if isRetry { + metadataFields++ + } + if errorMessage != "" { + metadataFields += 3 + } + if finalStatus != "" { + metadataFields++ + } + payload = msgp.AppendString(payload, "test_session_id") + payload = msgp.AppendUint64(payload, sessionID) + payload = msgp.AppendString(payload, "test_suite_id") + payload = msgp.AppendUint64(payload, suiteID) + payload = msgp.AppendString(payload, "span_id") + payload = msgp.AppendUint64(payload, spanID) + payload = msgp.AppendString(payload, "duration") + payload = msgp.AppendInt64(payload, int64(duration)) + payload = msgp.AppendString(payload, "meta") + payload = msgp.AppendMapHeader(payload, metadataFields) + payload = msgp.AppendString(payload, "test.name") + payload = msgp.AppendString(payload, name) + payload = msgp.AppendString(payload, "test.suite") + payload = msgp.AppendString(payload, suite) + payload = msgp.AppendString(payload, "test.status") + payload = msgp.AppendString(payload, status) + if finalStatus != "" { + payload = msgp.AppendString(payload, "test.final_status") + payload = msgp.AppendString(payload, finalStatus) + } + payload = msgp.AppendString(payload, "test.source.file") + payload = msgp.AppendString(payload, suite) + payload = msgp.AppendString(payload, "test.is_retry") + payload = msgp.AppendString(payload, strconv.FormatBool(isRetry)) + if isRetry { + payload = msgp.AppendString(payload, "test.retry_reason") + payload = msgp.AppendString(payload, "early_flake_detection") + } + if errorMessage != "" { + payload = msgp.AppendString(payload, "error.type") + payload = msgp.AppendString(payload, "AssertionError") + payload = msgp.AppendString(payload, "error.message") + payload = msgp.AppendString(payload, errorMessage) + payload = msgp.AppendString(payload, "error.stack") + payload = msgp.AppendString(payload, "stack trace") + } + payload = msgp.AppendString(payload, "metrics") + payload = msgp.AppendMapHeader(payload, 3) + payload = msgp.AppendString(payload, "test.is_retry") + if isRetry { + payload = msgp.AppendFloat64(payload, 1) + } else { + payload = msgp.AppendFloat64(payload, 0) + } + payload = msgp.AppendString(payload, "test.source.start") + payload = msgp.AppendFloat64(payload, 12) + payload = msgp.AppendString(payload, "test.source.end") + payload = msgp.AppendFloat64(payload, 14) + return payload +} diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go new file mode 100644 index 00000000..495f0ed3 --- /dev/null +++ b/internal/testdrive/intake/findings.go @@ -0,0 +1,403 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "fmt" + "slices" + "sort" + "time" +) + +const ( + minimumSlowDuration = 100 * time.Millisecond + minimumBroadCoverageFiles = 5 +) + +// TestFinding describes one observed test and its attempts. +type TestFinding struct { + Name string + Suite string + SourceFile string + SourceStart int + SourceEnd int + Status string + Duration time.Duration + Attempts []TestAttempt + CoverageLevel string + CoveredFiles []string +} + +// TestAttempt describes one observed run of a test. +type TestAttempt struct { + Status string + Duration time.Duration + Retry bool + RetryReason string + ErrorType string + ErrorMessage string + ErrorStack string +} + +// CoverageFinding describes one test or suite with unusually broad coverage. +type CoverageFinding struct { + Name string + Level string + SourceFile string + SourceStart int + SourceEnd int + FileCount int + Files []string +} + +// Findings contains the facts shown in the testdrive report. +type Findings struct { + TestCount int + TestEventCount int + CoveredTestCount int + TestDurationMedian time.Duration + CoveredFilesMedian int + CoverageLevel string + Tests []TestFinding + FailedTests []TestFinding + FlakyTests []TestFinding + SlowTests []TestFinding + BroadCoverage []CoverageFinding +} + +// Findings analyzes the test and coverage events captured by the intake. +func (s *Server) Findings() (Findings, error) { + tests, err := s.testReferences() + if err != nil { + return Findings{}, err + } + coverages, err := s.coverageReferences() + if err != nil { + return Findings{}, err + } + + findings := Findings{TestEventCount: len(tests), CoverageLevel: coverageLevel(coverages)} + findings.Tests, findings.FailedTests, findings.FlakyTests, findings.SlowTests, findings.TestDurationMedian = analyzeTests(tests) + addCoverageToTests(findings.Tests, tests, coverages, findings.CoverageLevel) + findings.TestCount = len(findings.Tests) + findings.CoveredTestCount = uniqueCoveredTestCount(tests, coverages) + findings.BroadCoverage, findings.CoveredFilesMedian = analyzeCoverage(tests, coverages, findings.CoverageLevel) + return findings, nil +} + +func analyzeTests(tests []testReference) ([]TestFinding, []TestFinding, []TestFinding, []TestFinding, time.Duration) { + testsByName := make(map[string][]testReference) + order := make([]string, 0) + for _, test := range tests { + key := testIdentity(test) + if _, found := testsByName[key]; !found { + order = append(order, key) + } + testsByName[key] = append(testsByName[key], test) + } + + failed := make([]TestFinding, 0) + flaky := make([]TestFinding, 0) + all := make([]TestFinding, 0, len(order)) + for _, key := range order { + attempts := testsByName[key] + finding := testFinding(attempts[0]) + finding.Attempts = make([]TestAttempt, 0, len(attempts)) + sawPass := false + sawFailure := false + for _, attempt := range attempts { + if attempt.finalStatus != "" { + finding.Status = attempt.finalStatus + finding.Duration = attempt.duration + } + sawPass = sawPass || attempt.status == "pass" || attempt.finalStatus == "pass" + sawFailure = sawFailure || attempt.status == "fail" || attempt.finalStatus == "fail" + finding.Attempts = append(finding.Attempts, TestAttempt{ + Status: attempt.status, + Duration: attempt.duration, + Retry: attempt.isRetry, + RetryReason: attempt.retryReason, + ErrorType: attempt.errorType, + ErrorMessage: attempt.errorMessage, + ErrorStack: attempt.errorStack, + }) + } + if finding.Status == "" { + finding.Status = attempts[len(attempts)-1].status + } + all = append(all, finding) + + if sawPass && sawFailure { + flaky = append(flaky, finding) + } else if finding.Status == "fail" { + failed = append(failed, finding) + } + } + + slow, median := slowTests(all) + sort.Slice(all, func(i, j int) bool { + if all[i].Suite != all[j].Suite { + return all[i].Suite < all[j].Suite + } + if all[i].SourceStart != all[j].SourceStart { + return all[i].SourceStart < all[j].SourceStart + } + return all[i].Name < all[j].Name + }) + sortTestFindings(failed) + sortTestFindings(flaky) + return all, failed, flaky, slow, median +} + +func addCoverageToTests(findings []TestFinding, tests []testReference, coverages []coverageReference, level string) { + testsBySpan := make(map[uint64]string, len(tests)) + for _, test := range tests { + testsBySpan[test.spanID] = testIdentity(test) + } + testFiles := make(map[string][]string) + suiteFiles := make(map[suiteReference][]string) + testCovered := make(map[string]bool) + suiteCovered := make(map[suiteReference]bool) + for _, coverage := range coverages { + if level == "test" && coverage.spanID != 0 { + if identity := testsBySpan[coverage.spanID]; identity != "" { + testCovered[identity] = true + testFiles[identity] = appendUnique(testFiles[identity], coverage.files...) + } + } + if level != "suite" || coverage.spanID != 0 { + continue + } + key := suiteReference{sessionID: coverage.sessionID, suiteID: coverage.suiteID} + suiteCovered[key] = true + suiteFiles[key] = appendUnique(suiteFiles[key], coverage.files...) + } + + for findingIndex := range findings { + finding := &findings[findingIndex] + identity := finding.Suite + "\x00" + finding.Name + if level == "test" && testCovered[identity] { + finding.CoveredFiles = appendUnique(finding.CoveredFiles, testFiles[identity]...) + finding.CoverageLevel = "test" + continue + } + if level != "suite" { + continue + } + for _, test := range tests { + if testIdentity(test) != identity { + continue + } + key := suiteReference{sessionID: test.sessionID, suiteID: test.suiteID} + finding.CoveredFiles = appendUnique(finding.CoveredFiles, suiteFiles[key]...) + if suiteCovered[key] { + finding.CoverageLevel = "suite" + } + } + } +} + +func appendUnique(values []string, additions ...string) []string { + for _, addition := range additions { + if slices.Contains(values, addition) { + continue + } + values = append(values, addition) + } + slices.Sort(values) + return values +} + +func slowTests(tests []TestFinding) ([]TestFinding, time.Duration) { + if len(tests) < 2 { + return nil, medianTestDuration(tests) + } + median := medianTestDuration(tests) + threshold := max(minimumSlowDuration, median*2) + + slow := make([]TestFinding, 0) + for _, test := range tests { + if test.Duration >= threshold && test.Duration > median { + slow = append(slow, test) + } + } + sortTestFindings(slow) + return slow, median +} + +func medianTestDuration(tests []TestFinding) time.Duration { + if len(tests) == 0 { + return 0 + } + durations := make([]time.Duration, 0, len(tests)) + for _, test := range tests { + durations = append(durations, test.Duration) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + middle := len(durations) / 2 + if len(durations)%2 == 1 { + return durations[middle] + } + return durations[middle-1] + (durations[middle]-durations[middle-1])/2 +} + +func coverageLevel(coverages []coverageReference) string { + level := "" + for _, coverage := range coverages { + if coverage.spanID != 0 { + return "test" + } + level = "suite" + } + return level +} + +func analyzeCoverage(tests []testReference, coverages []coverageReference, level string) ([]CoverageFinding, int) { + if len(coverages) < 2 || level == "" { + return nil, 0 + } + testsBySpan := make(map[uint64]testReference, len(tests)) + testsBySuite := make(map[suiteReference]testReference, len(tests)) + for _, test := range tests { + testsBySpan[test.spanID] = test + testsBySuite[suiteReference{sessionID: test.sessionID, suiteID: test.suiteID}] = test + } + + findingsByName := make(map[string]CoverageFinding, len(coverages)) + for _, coverage := range coverages { + if (level == "test" && coverage.spanID == 0) || (level == "suite" && coverage.spanID != 0) { + continue + } + finding := CoverageFinding{FileCount: coverage.fileCount, Files: coverage.files} + if coverage.spanID != 0 { + testReference, found := testsBySpan[coverage.spanID] + if !found { + continue + } + finding.Level = "test" + test := testFinding(testReference) + finding.Name = test.label() + finding.SourceFile = test.SourceFile + finding.SourceStart = test.SourceStart + finding.SourceEnd = test.SourceEnd + } else { + test, found := testsBySuite[suiteReference{sessionID: coverage.sessionID, suiteID: coverage.suiteID}] + if !found { + continue + } + finding.Level = "suite" + finding.Name = test.suite + finding.SourceFile = test.sourceFile + } + key := finding.Level + "\x00" + finding.Name + if current, found := findingsByName[key]; !found || finding.FileCount > current.FileCount { + findingsByName[key] = finding + } + } + if len(findingsByName) < 2 { + return nil, medianCoveredFiles(findingsByName) + } + + findings := make([]CoverageFinding, 0, len(findingsByName)) + fileCounts := make([]int, 0, len(findingsByName)) + for _, finding := range findingsByName { + findings = append(findings, finding) + fileCounts = append(fileCounts, finding.FileCount) + } + + sort.Ints(fileCounts) + median := medianInts(fileCounts) + threshold := max(minimumBroadCoverageFiles, median*2) + broad := make([]CoverageFinding, 0) + for _, finding := range findings { + if finding.FileCount >= threshold && finding.FileCount > median { + broad = append(broad, finding) + } + } + sort.Slice(broad, func(i, j int) bool { + if broad[i].FileCount == broad[j].FileCount { + return broad[i].Name < broad[j].Name + } + return broad[i].FileCount > broad[j].FileCount + }) + return broad, median +} + +func medianCoveredFiles(findings map[string]CoverageFinding) int { + fileCounts := make([]int, 0, len(findings)) + for _, finding := range findings { + fileCounts = append(fileCounts, finding.FileCount) + } + sort.Ints(fileCounts) + return medianInts(fileCounts) +} + +func medianInts(values []int) int { + if len(values) == 0 { + return 0 + } + middle := len(values) / 2 + if len(values)%2 == 1 { + return values[middle] + } + return values[middle-1] + (values[middle]-values[middle-1])/2 +} + +func uniqueCoveredTestCount(tests []testReference, coverages []coverageReference) int { + coveredTests := make(map[uint64]struct{}, len(coverages)) + coveredSuites := make(map[suiteReference]struct{}, len(coverages)) + for _, coverage := range coverages { + if coverage.spanID != 0 { + coveredTests[coverage.spanID] = struct{}{} + } else { + coveredSuites[suiteReference{sessionID: coverage.sessionID, suiteID: coverage.suiteID}] = struct{}{} + } + } + + coveredIdentities := make(map[string]struct{}) + for _, test := range tests { + _, testCovered := coveredTests[test.spanID] + _, suiteCovered := coveredSuites[suiteReference{sessionID: test.sessionID, suiteID: test.suiteID}] + if testCovered || suiteCovered { + coveredIdentities[testIdentity(test)] = struct{}{} + } + } + return len(coveredIdentities) +} + +func testIdentity(test testReference) string { + if test.name != "" || test.suite != "" { + return test.suite + "\x00" + test.name + } + return fmt.Sprintf("%d/%d", test.suiteID, test.spanID) +} + +func testFinding(test testReference) TestFinding { + name := test.name + if name == "" { + name = fmt.Sprintf("test %d", test.spanID) + } + return TestFinding{ + Name: name, Suite: test.suite, SourceFile: test.sourceFile, + SourceStart: test.sourceStart, SourceEnd: test.sourceEnd, Duration: test.duration, + } +} + +func (f TestFinding) label() string { + if f.Suite == "" { + return f.Name + } + return f.Suite + " › " + f.Name +} + +func sortTestFindings(findings []TestFinding) { + sort.Slice(findings, func(i, j int) bool { + if findings[i].Duration == findings[j].Duration { + return findings[i].label() < findings[j].label() + } + return findings[i].Duration > findings[j].Duration + }) +} diff --git a/internal/testdrive/intake/findings_test.go b/internal/testdrive/intake/findings_test.go new file mode 100644 index 00000000..bfcb4587 --- /dev/null +++ b/internal/testdrive/intake/findings_test.go @@ -0,0 +1,178 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestAnalyzeTestsFindsFailuresRetriesAndSlowTests(t *testing.T) { + tests := []testReference{ + {name: "fast", suite: "one.test.js", status: "pass", duration: 10 * time.Millisecond}, + {name: "flaky", suite: "one.test.js", status: "fail", duration: 20 * time.Millisecond}, + {name: "flaky", suite: "one.test.js", status: "pass", duration: 30 * time.Millisecond, isRetry: true}, + {name: "slow", suite: "two.test.js", status: "pass", duration: 400 * time.Millisecond}, + {name: "broken", suite: "two.test.js", status: "fail", duration: 15 * time.Millisecond}, + } + + all, failed, flaky, slow, median := analyzeTests(tests) + require.Len(t, all, 4) + require.Equal(t, []string{"fast", "flaky", "broken", "slow"}, []string{all[0].Name, all[1].Name, all[2].Name, all[3].Name}) + require.Equal(t, []TestFinding{{ + Name: "broken", Suite: "two.test.js", Status: "fail", Duration: 15 * time.Millisecond, + Attempts: []TestAttempt{{Status: "fail", Duration: 15 * time.Millisecond}}, + }}, failed) + require.Equal(t, []TestFinding{{ + Name: "flaky", Suite: "one.test.js", Status: "pass", Duration: 20 * time.Millisecond, + Attempts: []TestAttempt{ + {Status: "fail", Duration: 20 * time.Millisecond}, + {Status: "pass", Duration: 30 * time.Millisecond, Retry: true}, + }, + }}, flaky) + require.Equal(t, []TestFinding{{ + Name: "slow", Suite: "two.test.js", Status: "pass", Duration: 400 * time.Millisecond, + Attempts: []TestAttempt{{Status: "pass", Duration: 400 * time.Millisecond}}, + }}, slow) + require.Equal(t, 17500*time.Microsecond, median) +} + +func TestAnalyzeTestsUsesFinalStatusAndMarksMixedOutcomesFlaky(t *testing.T) { + tests := []testReference{ + {name: "flaky", suite: "one.test.js", status: "pass", finalStatus: "pass", duration: 10 * time.Millisecond}, + {name: "flaky", suite: "one.test.js", status: "fail", duration: 12 * time.Millisecond, isRetry: true}, + } + + all, failed, flaky, _, _ := analyzeTests(tests) + require.Empty(t, failed) + require.Len(t, flaky, 1) + require.Equal(t, "pass", flaky[0].Status) + require.Equal(t, 10*time.Millisecond, flaky[0].Duration) + require.Equal(t, "pass", all[0].Status) +} + +func TestAddCoverageToTestsUsesActiveCoverageLevel(t *testing.T) { + tests := []testReference{ + {sessionID: 1, suiteID: 10, spanID: 100, name: "one", suite: "one.test.js"}, + {sessionID: 1, suiteID: 10, spanID: 101, name: "two", suite: "one.test.js"}, + } + coverages := []coverageReference{ + {testReference: testReference{spanID: 100}, files: []string{"specific.js"}}, + {testReference: testReference{sessionID: 1, suiteID: 10}, files: []string{"shared.js"}}, + } + + t.Run("test", func(t *testing.T) { + findings, _, _, _, _ := analyzeTests(tests) + addCoverageToTests(findings, tests, coverages, "test") + require.Equal(t, "test", findings[0].CoverageLevel) + require.Equal(t, []string{"specific.js"}, findings[0].CoveredFiles) + require.Empty(t, findings[1].CoverageLevel) + }) + + t.Run("suite", func(t *testing.T) { + findings, _, _, _, _ := analyzeTests(tests) + addCoverageToTests(findings, tests, coverages, "suite") + for _, finding := range findings { + require.Equal(t, "suite", finding.CoverageLevel) + require.Equal(t, []string{"shared.js"}, finding.CoveredFiles) + } + }) +} + +func TestAnalyzeCoverageUsesActiveCoverageLevel(t *testing.T) { + tests := []testReference{ + {sessionID: 1, suiteID: 10, spanID: 100, name: "narrow", suite: "one.test.js"}, + {sessionID: 1, suiteID: 11, spanID: 110, name: "also narrow", suite: "one.test.js"}, + {sessionID: 1, suiteID: 20, spanID: 200, name: "broad", suite: "two.test.js"}, + {sessionID: 1, suiteID: 30, spanID: 300, name: "suite test", suite: "three.test.js"}, + } + coverages := []coverageReference{ + {testReference: testReference{spanID: 100}, fileCount: 1}, + {testReference: testReference{spanID: 110}, fileCount: 2}, + {testReference: testReference{spanID: 200}, fileCount: 12}, + {testReference: testReference{spanID: 999}, fileCount: 100}, + {testReference: testReference{sessionID: 1, suiteID: 10}, fileCount: 1}, + {testReference: testReference{sessionID: 1, suiteID: 20}, fileCount: 2}, + {testReference: testReference{sessionID: 1, suiteID: 30}, fileCount: 14}, + {fileCount: 100}, + } + + testFindings, testMedian := analyzeCoverage(tests, coverages, "test") + require.Equal(t, []CoverageFinding{ + {Name: "two.test.js › broad", Level: "test", FileCount: 12}, + }, testFindings) + require.Equal(t, 2, testMedian) + + suiteFindings, suiteMedian := analyzeCoverage(tests, coverages, "suite") + require.Equal(t, []CoverageFinding{ + {Name: "three.test.js", Level: "suite", FileCount: 14}, + }, suiteFindings) + require.Equal(t, 2, suiteMedian) +} + +func TestUniqueCoveredTestCountDoesNotCountRetriesTwice(t *testing.T) { + tests := []testReference{ + {sessionID: 1, suiteID: 10, spanID: 100, name: "flaky", suite: "one.test.js"}, + {sessionID: 1, suiteID: 10, spanID: 101, name: "flaky", suite: "one.test.js", isRetry: true}, + {sessionID: 1, suiteID: 10, spanID: 102, name: "stable", suite: "one.test.js"}, + } + coverages := []coverageReference{ + {testReference: testReference{spanID: 100}}, + {testReference: testReference{spanID: 101}}, + {testReference: testReference{spanID: 102}}, + } + + require.Equal(t, 2, uniqueCoveredTestCount(tests, coverages)) +} + +func TestAnalyzeCoverageUsesLargestDuplicateAndReportsSingleMedian(t *testing.T) { + tests := []testReference{{spanID: 100, name: "one", suite: "one.test.js"}} + coverages := []coverageReference{ + {testReference: testReference{spanID: 100}, fileCount: 2}, + {testReference: testReference{spanID: 100}, fileCount: 7}, + } + + broad, median := analyzeCoverage(tests, coverages, "test") + require.Empty(t, broad) + require.Equal(t, 7, median) +} + +func TestAnalyzeTestsHandlesMissingNamesAndStableOrdering(t *testing.T) { + tests := []testReference{ + {suiteID: 2, spanID: 20, status: "fail", duration: time.Second}, + {suiteID: 1, spanID: 10, status: "fail", duration: time.Second}, + {name: "named", status: "pass", duration: time.Millisecond}, + } + + all, failed, _, _, _ := analyzeTests(tests) + require.Equal(t, []string{"named", "test 10", "test 20"}, []string{all[0].Name, all[1].Name, all[2].Name}) + require.Equal(t, []string{"test 10", "test 20"}, []string{failed[0].Name, failed[1].Name}) + require.Equal(t, "named", all[0].label()) +} + +func TestMedianHelpers(t *testing.T) { + require.Equal(t, 0, medianInts(nil)) + require.Equal(t, 4, medianInts([]int{4})) + require.Equal(t, 4, medianInts([]int{2, 6})) + require.Equal(t, 6, medianInts([]int{2, 6, 9})) + require.Equal(t, 0, medianCoveredFiles(nil)) + require.Equal(t, 5, medianCoveredFiles(map[string]CoverageFinding{ + "one": {FileCount: 3}, + "two": {FileCount: 7}, + })) +} + +func TestCoverageLevelAndAppendUnique(t *testing.T) { + require.Empty(t, coverageLevel(nil)) + require.Equal(t, "suite", coverageLevel([]coverageReference{{testReference: testReference{suiteID: 1}}})) + require.Equal(t, "test", coverageLevel([]coverageReference{ + {testReference: testReference{suiteID: 1}}, + {testReference: testReference{spanID: 2}}, + })) + require.Equal(t, []string{"a.js", "b.js"}, appendUnique([]string{"b.js"}, "a.js", "b.js")) +} diff --git a/internal/testdrive/intake/server.go b/internal/testdrive/intake/server.go new file mode 100644 index 00000000..901fbe70 --- /dev/null +++ b/internal/testdrive/intake/server.go @@ -0,0 +1,298 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +// Package intake provides the local HTTP intake used by testdrive. +package intake + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "github.com/tinylib/msgp/msgp" +) + +const ( + shutdownTimeout = 5 * time.Second + intakeDirectoryName = "intake" +) + +// RawRequest is an HTTP request observed by the local intake. +type RawRequest struct { + Method string + Path string + Header http.Header + Body []byte +} + +type storedRequest struct { + Method string `json:"method"` + Path string `json:"path"` + ContentType string `json:"content_type,omitempty"` + Body json.RawMessage `json:"body"` +} + +type storedMultipartPart struct { + Name string `json:"name"` + Filename string `json:"filename,omitempty"` + ContentType string `json:"content_type,omitempty"` + Body json.RawMessage `json:"body"` +} + +// Server is a local HTTP intake. +type Server struct { + server *http.Server + url string + done chan error + closeOnce sync.Once + closeErr error + directory string + + requestsMu sync.Mutex + requests []RawRequest +} + +// Start starts an HTTP intake on a kernel-assigned loopback port and stores +// every request under the testdrive session directory. +func Start(sessionDirectory string) (*Server, error) { + intakeDirectory := filepath.Join(sessionDirectory, intakeDirectoryName) + if err := os.MkdirAll(intakeDirectory, 0755); err != nil { + return nil, fmt.Errorf("create local testdrive intake directory: %w", err) + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, fmt.Errorf("listen for local testdrive intake: %w", err) + } + + server := &Server{ + url: "http://" + listener.Addr().String(), + done: make(chan error, 1), + directory: intakeDirectory, + } + httpServer := &http.Server{ + Handler: server.recordRequests(newHandler()), + ReadHeaderTimeout: 10 * time.Second, + } + server.server = httpServer + + go func() { + serveErr := httpServer.Serve(listener) + if errors.Is(serveErr, http.ErrServerClosed) { + serveErr = nil + } + server.done <- serveErr + close(server.done) + }() + + return server, nil +} + +// URL returns the base URL of the running intake. +func (s *Server) URL() string { + return s.url +} + +// Requests returns a snapshot of the raw requests observed by the intake. +func (s *Server) Requests() []RawRequest { + s.requestsMu.Lock() + defer s.requestsMu.Unlock() + + requests := make([]RawRequest, len(s.requests)) + for i, request := range s.requests { + requests[i] = RawRequest{ + Method: request.Method, + Path: request.Path, + Header: request.Header.Clone(), + Body: slices.Clone(request.Body), + } + } + return requests +} + +func (s *Server) recordRequests(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + body, err := io.ReadAll(request.Body) + if err != nil { + http.Error(w, "read request body", http.StatusBadRequest) + return + } + _ = request.Body.Close() + request.Body = io.NopCloser(bytes.NewReader(body)) + + rawRequest := RawRequest{ + Method: request.Method, + Path: request.URL.Path, + Header: request.Header.Clone(), + Body: slices.Clone(body), + } + s.requestsMu.Lock() + requestNumber := len(s.requests) + 1 + persistErr := s.persistRequest(requestNumber, rawRequest) + s.requests = append(s.requests, rawRequest) + s.requestsMu.Unlock() + if persistErr != nil { + http.Error(w, "save request", http.StatusInternalServerError) + return + } + + next.ServeHTTP(w, request) + }) +} + +func (s *Server) persistRequest(number int, request RawRequest) error { + body, err := uncompressRequestBody(request) + if err != nil { + return err + } + decodedBody, err := decodeRequestBody(body, request.Header.Get("Content-Type")) + if err != nil { + return fmt.Errorf("decode request body: %w", err) + } + stored := storedRequest{ + Method: request.Method, + Path: request.Path, + ContentType: request.Header.Get("Content-Type"), + Body: decodedBody, + } + encoded, err := json.MarshalIndent(stored, "", " ") + if err != nil { + return fmt.Errorf("encode request: %w", err) + } + encoded = append(encoded, '\n') + filename := fmt.Sprintf("%03d-%s.json", number, requestFileLabel(request.Path)) + if err := os.WriteFile(filepath.Join(s.directory, filename), encoded, 0644); err != nil { + return fmt.Errorf("write request: %w", err) + } + return nil +} + +func uncompressRequestBody(request RawRequest) ([]byte, error) { + contentEncoding := strings.TrimSpace(request.Header.Get("Content-Encoding")) + if contentEncoding == "" || strings.EqualFold(contentEncoding, "identity") { + return request.Body, nil + } + if !strings.EqualFold(contentEncoding, "gzip") { + return nil, fmt.Errorf("unsupported content encoding %q", contentEncoding) + } + + reader, err := gzip.NewReader(bytes.NewReader(request.Body)) + if err != nil { + return nil, fmt.Errorf("open gzip request body: %w", err) + } + body, readErr := io.ReadAll(reader) + closeErr := reader.Close() + if err := errors.Join(readErr, closeErr); err != nil { + return nil, fmt.Errorf("read gzip request body: %w", err) + } + return body, nil +} + +func decodeRequestBody(body []byte, contentType string) (json.RawMessage, error) { + mediaType, params, _ := mime.ParseMediaType(contentType) + switch mediaType { + case "application/json": + if !json.Valid(body) { + return nil, fmt.Errorf("invalid JSON") + } + return slices.Clone(body), nil + case "application/msgpack", "application/x-msgpack": + return decodeMessagePack(body) + case "multipart/form-data": + return decodeMultipart(body, params["boundary"]) + default: + return json.Marshal(string(body)) + } +} + +func decodeMessagePack(body []byte) (json.RawMessage, error) { + decoded, rest, err := msgp.ReadIntfBytes(body) + if err != nil { + return nil, err + } + if len(rest) != 0 { + return nil, fmt.Errorf("unexpected trailing MessagePack bytes") + } + return json.Marshal(decoded) +} + +func decodeMultipart(body []byte, boundary string) (json.RawMessage, error) { + reader := multipart.NewReader(bytes.NewReader(body), boundary) + parts := make([]storedMultipartPart, 0) + for { + part, err := reader.NextPart() + if err == io.EOF { + return json.Marshal(struct { + Parts []storedMultipartPart `json:"parts"` + }{Parts: parts}) + } + if err != nil { + return nil, err + } + + partBody, readErr := io.ReadAll(part) + _ = part.Close() + if readErr != nil { + return nil, readErr + } + contentType := part.Header.Get("Content-Type") + decodedBody, decodeErr := decodeRequestBody(partBody, contentType) + if decodeErr != nil { + return nil, decodeErr + } + parts = append(parts, storedMultipartPart{ + Name: part.FormName(), + Filename: part.FileName(), + ContentType: contentType, + Body: decodedBody, + }) + } +} + +func requestFileLabel(requestPath string) string { + switch requestPath { + case settingsPath: + return "settings" + case testCyclePath: + return "citestcycle" + case testCoveragePath: + return "citestcov" + case knownTestsPath: + return "known-tests" + case skippableTestsPath: + return "skippable-tests" + case testManagementPath: + return "test-management" + default: + return "request" + } +} + +// Close stops the intake and waits for its server goroutine to finish. +func (s *Server) Close() error { + s.closeOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + + shutdownErr := s.server.Shutdown(ctx) + serveErr := <-s.done + s.closeErr = errors.Join(shutdownErr, serveErr) + }) + return s.closeErr +} diff --git a/internal/testdrive/intake/server_test.go b/internal/testdrive/intake/server_test.go new file mode 100644 index 00000000..dae902f3 --- /dev/null +++ b/internal/testdrive/intake/server_test.go @@ -0,0 +1,289 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "mime/multipart" + "net" + "net/http" + "net/textproto" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tinylib/msgp/msgp" +) + +func TestStart(t *testing.T) { + server, err := Start(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, server.Close()) + }) + + serverURL, err := url.Parse(server.URL()) + require.NoError(t, err) + require.Equal(t, "http", serverURL.Scheme) + host, port, err := net.SplitHostPort(serverURL.Host) + require.NoError(t, err) + require.Equal(t, "127.0.0.1", host) + require.NotEqual(t, "0", port) + + response, err := testHTTPClient().Get(server.URL()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, response.Body.Close()) + }) + require.Equal(t, http.StatusNotFound, response.StatusCode) +} + +func TestCloseStopsServer(t *testing.T) { + server, err := Start(t.TempDir()) + require.NoError(t, err) + + require.NoError(t, server.Close()) + require.NoError(t, server.Close()) + + _, err = testHTTPClient().Get(server.URL()) + require.Error(t, err) + + _, running := <-server.done + require.False(t, running, "server goroutine did not finish") +} + +func TestStartSupportsSimultaneousServers(t *testing.T) { + first, err := Start(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, first.Close()) + }) + + second, err := Start(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, second.Close()) + }) + + require.NotEqual(t, first.URL(), second.URL()) + for _, server := range []*Server{first, second} { + response, requestErr := testHTTPClient().Get(server.URL()) + require.NoError(t, requestErr) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusNotFound, response.StatusCode) + } +} + +func TestServerCapturesRawRequests(t *testing.T) { + sessionDirectory := t.TempDir() + server, err := Start(sessionDirectory) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, server.Close()) + }) + + request, err := http.NewRequest(http.MethodPost, server.URL()+"/observed", bytes.NewBufferString("raw body")) + require.NoError(t, err) + request.Header.Set("Content-Type", "application/octet-stream") + response, err := testHTTPClient().Do(request) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, response.Body.Close()) + }) + require.Equal(t, http.StatusOK, response.StatusCode) + + requests := server.Requests() + require.Len(t, requests, 1) + require.Equal(t, http.MethodPost, requests[0].Method) + require.Equal(t, "/observed", requests[0].Path) + require.Equal(t, "application/octet-stream", requests[0].Header.Get("Content-Type")) + require.Equal(t, []byte("raw body"), requests[0].Body) + + storedBytes, err := os.ReadFile(filepath.Join(sessionDirectory, intakeDirectoryName, "001-request.json")) + require.NoError(t, err) + var stored storedRequest + require.NoError(t, json.Unmarshal(storedBytes, &stored)) + require.Equal(t, http.MethodPost, stored.Method) + require.Equal(t, "/observed", stored.Path) + var storedBody string + require.NoError(t, json.Unmarshal(stored.Body, &storedBody)) + require.Equal(t, "raw body", storedBody) +} + +func TestServerStoresMessagePackAsJSON(t *testing.T) { + sessionDirectory := t.TempDir() + server, err := Start(sessionDirectory) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, server.Close()) + }) + + payload := msgp.AppendMapHeader(nil, 1) + payload = msgp.AppendString(payload, "events") + payload = msgp.AppendArrayHeader(payload, 0) + response, err := testHTTPClient().Post(server.URL()+testCyclePath, "application/msgpack", bytes.NewReader(payload)) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusOK, response.StatusCode) + + intakeDirectory := filepath.Join(sessionDirectory, intakeDirectoryName) + files, err := os.ReadDir(intakeDirectory) + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, "001-citestcycle.json", files[0].Name()) + storedBytes, err := os.ReadFile(filepath.Join(intakeDirectory, files[0].Name())) + require.NoError(t, err) + require.True(t, json.Valid(storedBytes)) + require.Contains(t, string(storedBytes), `"events": []`) +} + +func TestServerStoresAndRecognizesGzippedMessagePack(t *testing.T) { + sessionDirectory := t.TempDir() + server, err := Start(sessionDirectory) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, server.Close()) + }) + + payload := msgp.AppendMapHeader(nil, 1) + payload = msgp.AppendString(payload, "events") + payload = msgp.AppendArrayHeader(payload, 1) + payload = appendEvent(payload, "test", 10, 20, 30) + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err = writer.Write(payload) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + request, err := http.NewRequest(http.MethodPost, server.URL()+testCyclePath, bytes.NewReader(compressed.Bytes())) + require.NoError(t, err) + request.Header.Set("Content-Type", "application/msgpack") + request.Header.Set("Content-Encoding", "gzip") + response, err := testHTTPClient().Do(request) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusOK, response.StatusCode) + + testCount, err := server.TestEventCount() + require.NoError(t, err) + require.Equal(t, 1, testCount) + + storedBytes, err := os.ReadFile(filepath.Join(sessionDirectory, intakeDirectoryName, "001-citestcycle.json")) + require.NoError(t, err) + require.True(t, json.Valid(storedBytes)) + require.Contains(t, string(storedBytes), `"type": "test"`) +} + +func TestStartRejectsInvalidSessionDirectory(t *testing.T) { + sessionPath := filepath.Join(t.TempDir(), "session-file") + require.NoError(t, os.WriteFile(sessionPath, []byte("not a directory"), 0644)) + + _, err := Start(sessionPath) + require.ErrorContains(t, err, "create local testdrive intake directory") +} + +func TestDecodeMultipartStoresEveryPartAsJSON(t *testing.T) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + jsonHeader := textproto.MIMEHeader{} + jsonHeader.Set("Content-Disposition", `form-data; name="metadata"`) + jsonHeader.Set("Content-Type", "application/json") + jsonPart, err := writer.CreatePart(jsonHeader) + require.NoError(t, err) + _, err = jsonPart.Write([]byte(`{"framework":"jest"}`)) + require.NoError(t, err) + + msgpackHeader := textproto.MIMEHeader{} + msgpackHeader.Set("Content-Disposition", `form-data; name="events"; filename="events.msgpack"`) + msgpackHeader.Set("Content-Type", "application/msgpack") + msgpackPart, err := writer.CreatePart(msgpackHeader) + require.NoError(t, err) + payload := msgp.AppendMapHeader(nil, 1) + payload = msgp.AppendString(payload, "count") + payload = msgp.AppendInt(payload, 2) + _, err = msgpackPart.Write(payload) + require.NoError(t, err) + + plainPart, err := writer.CreateFormField("note") + require.NoError(t, err) + _, err = plainPart.Write([]byte("hello")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + decoded, err := decodeRequestBody(body.Bytes(), writer.FormDataContentType()) + require.NoError(t, err) + var stored struct { + Parts []storedMultipartPart `json:"parts"` + } + require.NoError(t, json.Unmarshal(decoded, &stored)) + require.Len(t, stored.Parts, 3) + require.Equal(t, "metadata", stored.Parts[0].Name) + require.JSONEq(t, `{"framework":"jest"}`, string(stored.Parts[0].Body)) + require.Equal(t, "events.msgpack", stored.Parts[1].Filename) + require.JSONEq(t, `{"count":2}`, string(stored.Parts[1].Body)) + require.JSONEq(t, `"hello"`, string(stored.Parts[2].Body)) +} + +func TestRequestDecodingRejectsMalformedPayloads(t *testing.T) { + tests := []struct { + name string + body []byte + contentType string + errorText string + }{ + {name: "JSON", body: []byte("{"), contentType: "application/json", errorText: "invalid JSON"}, + {name: "MessagePack", body: []byte{0xc1}, contentType: "application/msgpack", errorText: "msgp"}, + {name: "trailing MessagePack", body: append(msgp.AppendInt(nil, 1), 0), contentType: "application/x-msgpack", errorText: "trailing"}, + {name: "multipart", body: []byte("--unfinished"), contentType: "multipart/form-data; boundary=boundary", errorText: "EOF"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := decodeRequestBody(test.body, test.contentType) + require.ErrorContains(t, err, test.errorText) + }) + } +} + +func TestUncompressRequestBodyHandlesIdentityAndErrors(t *testing.T) { + body := []byte("plain") + for _, encoding := range []string{"", " identity "} { + decoded, err := uncompressRequestBody(RawRequest{Header: http.Header{"Content-Encoding": {encoding}}, Body: body}) + require.NoError(t, err) + require.Equal(t, body, decoded) + } + + _, err := uncompressRequestBody(RawRequest{Header: http.Header{"Content-Encoding": {"br"}}, Body: body}) + require.ErrorContains(t, err, "unsupported content encoding") + + _, err = uncompressRequestBody(RawRequest{Header: http.Header{"Content-Encoding": {"gzip"}}, Body: body}) + require.ErrorContains(t, err, "open gzip request body") +} + +func TestRequestFileLabel(t *testing.T) { + tests := map[string]string{ + settingsPath: "settings", + testCyclePath: "citestcycle", + testCoveragePath: "citestcov", + knownTestsPath: "known-tests", + skippableTestsPath: "skippable-tests", + testManagementPath: "test-management", + "/other": "request", + } + for path, expected := range tests { + require.Equal(t, expected, requestFileLabel(path)) + } +} + +func testHTTPClient() *http.Client { + return &http.Client{Timeout: time.Second} +} diff --git a/internal/testdrive/intake/settings.go b/internal/testdrive/intake/settings.go new file mode 100644 index 00000000..aab23bac --- /dev/null +++ b/internal/testdrive/intake/settings.go @@ -0,0 +1,155 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "encoding/json" + "net/http" +) + +const ( + settingsPath = "/api/v2/libraries/tests/services/setting" + knownTestsPath = "/api/v2/ci/libraries/tests" + skippableTestsPath = "/api/v2/ci/tests/skippable" + testManagementPath = "/api/v2/test/libraries/test-management/tests" + settingsResponseID = "test-settings" + settingsResponseType = "ci_app_test_service_libraries_settings" + testdriveCorrelationID = "ddtest-testdrive" +) + +type settingsRequest struct { + Data struct { + ID string `json:"id"` + } `json:"data"` +} + +type settingsResponse struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes settingsAttributes `json:"attributes"` + } `json:"data"` +} + +type settingsAttributes struct { + CodeCoverage bool `json:"code_coverage"` + CoverageReportUploadEnabled bool `json:"coverage_report_upload_enabled"` + TestsSkipping bool `json:"tests_skipping"` + RequireGit bool `json:"require_git"` + ITREnabled bool `json:"itr_enabled"` + ImpactedTestsEnabled bool `json:"impacted_tests_enabled"` + FlakyTestRetriesEnabled bool `json:"flaky_test_retries_enabled"` + DIEnabled bool `json:"di_enabled"` + KnownTestsEnabled bool `json:"known_tests_enabled"` + EarlyFlakeDetection earlyFlakeDetectionSettings `json:"early_flake_detection"` + TestManagement testManagementSettings `json:"test_management"` +} + +type earlyFlakeDetectionSettings struct { + Enabled bool `json:"enabled"` + SlowTestRetries map[string]int `json:"slow_test_retries"` + FaultySessionThreshold int `json:"faulty_session_threshold"` +} + +type testManagementSettings struct { + Enabled bool `json:"enabled"` + AttemptToFixRetries int `json:"attempt_to_fix_retries"` +} + +func newHandler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("POST "+settingsPath, handleSettings) + mux.HandleFunc("POST "+knownTestsPath, handleKnownTests) + mux.HandleFunc("POST "+skippableTestsPath, handleSkippableTests) + mux.HandleFunc("POST "+testManagementPath, handleTestManagement) + mux.HandleFunc("/", func(w http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodPost { + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, request) + }) + return mux +} + +func handleSettings(w http.ResponseWriter, request *http.Request) { + var settingsRequest settingsRequest + if err := json.NewDecoder(request.Body).Decode(&settingsRequest); err != nil { + http.Error(w, "invalid settings request", http.StatusBadRequest) + return + } + + responseID := settingsRequest.Data.ID + if responseID == "" { + responseID = settingsResponseID + } + + response := settingsResponse{} + response.Data.ID = responseID + response.Data.Type = settingsResponseType + response.Data.Attributes = settingsAttributes{ + CodeCoverage: true, + CoverageReportUploadEnabled: true, + TestsSkipping: true, + ITREnabled: true, + ImpactedTestsEnabled: true, + FlakyTestRetriesEnabled: true, + DIEnabled: true, + KnownTestsEnabled: true, + EarlyFlakeDetection: earlyFlakeDetectionSettings{ + Enabled: true, + SlowTestRetries: map[string]int{"5s": 1, "10s": 1, "30s": 1, "5m": 1}, + FaultySessionThreshold: 100, + }, + TestManagement: testManagementSettings{ + Enabled: true, + AttemptToFixRetries: 1, + }, + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(response) +} + +func handleKnownTests(w http.ResponseWriter, _ *http.Request) { + // JavaScript tracers validate the module entry before enabling EFD. Return + // an empty dataset for every supported runner, preserving Jest's retry loop. + tests := make(map[string]any) + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest", "rspec", "minitest"} { + tests[name] = map[string]any{} + } + writeJSON(w, map[string]any{ + "data": map[string]any{ + "attributes": map[string]any{ + "tests": tests, + }, + }, + }) +} + +func handleSkippableTests(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{ + "data": []any{}, + "meta": map[string]any{ + "correlation_id": testdriveCorrelationID, + "coverage": map[string]any{}, + }, + }) +} + +func handleTestManagement(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{ + "data": map[string]any{ + "attributes": map[string]any{"modules": map[string]any{}}, + }, + }) +} + +func writeJSON(w http.ResponseWriter, response any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(response) +} diff --git a/internal/testdrive/intake/settings_test.go b/internal/testdrive/intake/settings_test.go new file mode 100644 index 00000000..cf973fa3 --- /dev/null +++ b/internal/testdrive/intake/settings_test.go @@ -0,0 +1,81 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSettingsEnablesTestOptimizationCoverage(t *testing.T) { + server, err := Start(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, server.Close()) + }) + + requestBody := bytes.NewBufferString(`{"data":{"id":"request-123"}}`) + response, err := testHTTPClient().Post(server.URL()+settingsPath, "application/json", requestBody) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, response.Body.Close()) + }) + require.Equal(t, http.StatusAccepted, response.StatusCode) + require.Equal(t, "application/json", response.Header.Get("Content-Type")) + + var settings settingsResponse + require.NoError(t, json.NewDecoder(response.Body).Decode(&settings)) + require.Equal(t, "request-123", settings.Data.ID) + require.Equal(t, settingsResponseType, settings.Data.Type) + require.True(t, settings.Data.Attributes.ITREnabled) + require.True(t, settings.Data.Attributes.CodeCoverage) + require.True(t, settings.Data.Attributes.TestsSkipping) + require.False(t, settings.Data.Attributes.RequireGit) + require.True(t, settings.Data.Attributes.CoverageReportUploadEnabled) + require.True(t, settings.Data.Attributes.ImpactedTestsEnabled) + require.True(t, settings.Data.Attributes.FlakyTestRetriesEnabled) + require.True(t, settings.Data.Attributes.DIEnabled) + require.True(t, settings.Data.Attributes.KnownTestsEnabled) + require.True(t, settings.Data.Attributes.EarlyFlakeDetection.Enabled) + require.Equal(t, 1, settings.Data.Attributes.EarlyFlakeDetection.SlowTestRetries["5s"]) + require.True(t, settings.Data.Attributes.TestManagement.Enabled) +} + +func TestAdvancedFeatureEndpointsReturnSafeEmptyDatasets(t *testing.T) { + server, err := Start(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, server.Close()) + }) + + tests := []struct { + path string + contains string + }{ + {path: knownTestsPath, contains: `"jest":{}`}, + {path: skippableTestsPath, contains: `"data":[]`}, + {path: testManagementPath, contains: `"modules":{}`}, + } + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + response, err := testHTTPClient().Post(server.URL()+test.path, "application/json", bytes.NewBufferString(`{"data":{}}`)) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, response.Body.Close()) + }) + require.Equal(t, http.StatusOK, response.StatusCode) + + var body bytes.Buffer + _, err = body.ReadFrom(response.Body) + require.NoError(t, err) + require.Contains(t, body.String(), test.contains) + }) + } +} diff --git a/internal/testdrive/intake/testcoverage.go b/internal/testdrive/intake/testcoverage.go new file mode 100644 index 00000000..258d774e --- /dev/null +++ b/internal/testdrive/intake/testcoverage.go @@ -0,0 +1,178 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package intake + +import ( + "bytes" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + + "github.com/tinylib/msgp/msgp" +) + +const testCoveragePath = "/api/v2/citestcov" + +type coverageReference struct { + testReference + fileCount int + files []string +} + +// CoveredTestCount returns the number of observed tests with matching coverage. +func (s *Server) CoveredTestCount() (int, error) { + tests, err := s.testReferences() + if err != nil { + return 0, err + } + + coverages, err := s.coverageReferences() + if err != nil { + return 0, err + } + coveredTests := make(map[uint64]struct{}, len(coverages)) + coveredSuites := make(map[suiteReference]struct{}, len(coverages)) + for _, coverage := range coverages { + if coverage.spanID != 0 { + coveredTests[coverage.spanID] = struct{}{} + continue + } + coveredSuites[suiteReference{sessionID: coverage.sessionID, suiteID: coverage.suiteID}] = struct{}{} + } + + count := 0 + for _, test := range tests { + _, testCovered := coveredTests[test.spanID] + _, suiteCovered := coveredSuites[suiteReference{sessionID: test.sessionID, suiteID: test.suiteID}] + if testCovered || suiteCovered { + count++ + } + } + return count, nil +} + +func (s *Server) coverageReferences() ([]coverageReference, error) { + var coverages []coverageReference + for _, request := range s.Requests() { + if request.Method != http.MethodPost || request.Path != testCoveragePath { + continue + } + + requestCoverages, err := readMultipartCoverage(request) + if err != nil { + return nil, fmt.Errorf("recognize coverage in %s: %w", testCoveragePath, err) + } + coverages = append(coverages, requestCoverages...) + } + return coverages, nil +} + +func readMultipartCoverage(request RawRequest) ([]coverageReference, error) { + mediaType, params, err := mime.ParseMediaType(request.Header.Get("Content-Type")) + if err != nil { + return nil, err + } + if mediaType != "multipart/form-data" { + return nil, fmt.Errorf("unexpected content type %q", mediaType) + } + + body, err := uncompressRequestBody(request) + if err != nil { + return nil, err + } + reader := multipart.NewReader(bytes.NewReader(body), params["boundary"]) + var coverages []coverageReference + for { + part, partErr := reader.NextPart() + if partErr == io.EOF { + return coverages, nil + } + if partErr != nil { + return nil, partErr + } + + partBody, readErr := io.ReadAll(part) + _ = part.Close() + if readErr != nil { + return nil, readErr + } + partMediaType, _, parseErr := mime.ParseMediaType(part.Header.Get("Content-Type")) + if parseErr != nil || partMediaType != "application/msgpack" { + continue + } + + partCoverages, decodeErr := readCoverageEntries(partBody) + if decodeErr != nil { + return nil, decodeErr + } + coverages = append(coverages, partCoverages...) + } +} + +func readCoverageEntries(payload []byte) ([]coverageReference, error) { + fieldCount, rest, err := msgp.ReadMapHeaderBytes(payload) + if err != nil { + return nil, err + } + + var coverages []coverageReference + for range fieldCount { + var field string + field, rest, err = msgp.ReadStringBytes(rest) + if err != nil { + return nil, err + } + if field != "coverages" { + rest, err = msgp.Skip(rest) + if err != nil { + return nil, err + } + continue + } + + var entryCount uint32 + entryCount, rest, err = msgp.ReadArrayHeaderBytes(rest) + if err != nil { + return nil, err + } + for range entryCount { + var coverage coverageReference + coverage, rest, err = readCoverageReference(rest) + if err != nil { + return nil, err + } + coverages = append(coverages, coverage) + } + } + return coverages, nil +} + +func readCoverageReference(payload []byte) (coverageReference, []byte, error) { + content, rest, err := msgp.ReadMapStrIntfBytes(payload, nil) + if err != nil { + return coverageReference{}, nil, err + } + + encodedFiles, _ := content["files"].([]any) + files := make([]string, 0, len(encodedFiles)) + for _, encodedFile := range encodedFiles { + file, _ := encodedFile.(map[string]any) + if filename := text(file["filename"]); filename != "" { + files = append(files, filename) + } + } + return coverageReference{ + testReference: testReference{ + sessionID: unsigned(content["test_session_id"]), + suiteID: unsigned(content["test_suite_id"]), + spanID: unsigned(content["span_id"]), + }, + fileCount: len(encodedFiles), + files: files, + }, rest, nil +} diff --git a/internal/testdrive/multiframework_test.go b/internal/testdrive/multiframework_test.go new file mode 100644 index 00000000..62896dcf --- /dev/null +++ b/internal/testdrive/multiframework_test.go @@ -0,0 +1,111 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/DataDog/ddtest/internal/testdrive/intake" + "github.com/stretchr/testify/require" +) + +func TestPrepareAllSupportedFrameworks(t *testing.T) { + for _, name := range []string{"jest", "mocha", "vitest", "playwright", "cucumber", "cypress", "pytest", "rspec", "minitest"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + switch name { + case "pytest": + requireWriteFile(t, filepath.Join(root, "pyproject.toml"), "[tool.pytest.ini_options]\n") + case "rspec", "minitest": + requireWriteFile(t, filepath.Join(root, "Gemfile"), "gem '"+name+"'\n") + default: + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"`+name+`"}}`) + } + run, err := Prepare(root) + require.NoError(t, err) + var preview bytes.Buffer + run.Preview(&preview) + require.Contains(t, preview.String(), displayName(name)) + _, err = os.Stat(filepath.Join(root, ".testoptimization")) + require.True(t, os.IsNotExist(err), "preview must be read-only") + if name == "cypress" { + return + } // Browser wrapper has its own real-run test. + run.tracer = &fakeTracer{preloadPath: filepath.Join(root, "isolated")} + executor := &fakeTestdriveExecutor{} + run.executor = executor + run.startIntake = func(string) (localIntake, error) { + return &fakeIntake{url: "http://127.0.0.1:1234", findings: intake.Findings{TestEventCount: 1, TestCount: 1}}, nil + } + var output bytes.Buffer + require.NoError(t, run.Run(t.Context(), &output)) + require.Contains(t, output.String(), displayName(name)+": Passed") + require.Contains(t, output.String(), "Tests with coverage: 0 / 1") + require.Equal(t, "ddtest-testdrive", executor.env["DD_API_KEY"]) + if name == "cucumber" { + require.Equal(t, "false", executor.env["DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"]) + } + switch run.language { + case "javascript": + require.Contains(t, executor.env["NODE_OPTIONS"], "--import") + case "python": + require.NotEmpty(t, executor.env["PYTHONPATH"]) + require.Contains(t, executor.env["PYTEST_ADDOPTS"], "--ddtrace") + require.NotContains(t, executor.env, "NODE_OPTIONS") + case "ruby": + require.NotEmpty(t, executor.env["BUNDLE_GEMFILE"]) + require.Contains(t, executor.env["RUBYOPT"], "datadog/ci/auto_instrument") + require.NotContains(t, executor.env, "NODE_OPTIONS") + } + }) + } +} + +func TestPrepareRequiresSelectionForMultipleFrameworks(t *testing.T) { + root := t.TempDir() + requireWriteFile(t, filepath.Join(root, "package.json"), `{"scripts":{"test":"vitest run","e2e":"playwright test"}}`) + _, err := Prepare(root) + require.ErrorContains(t, err, "--framework") + run, err := PrepareWithFramework(root, "vitest") + require.NoError(t, err) + require.Equal(t, "vitest", run.framework.Name()) + _, err = PrepareWithFramework(root, "pytest") + require.ErrorContains(t, err, "could not detect") +} + +func TestLanguageEnvironmentsPreserveCustomerOptions(t *testing.T) { + t.Setenv("PYTHONPATH", "/customer/modules") + t.Setenv("PYTEST_ADDOPTS", "-q") + t.Setenv("RUBYOPT", "-W0") + python := (&Testdrive{language: "python"}).environment("/session/python", "http://127.0.0.1:1234", "session") + require.Equal(t, "/session/python"+string(os.PathListSeparator)+"/customer/modules", python["PYTHONPATH"]) + require.Equal(t, "-q --ddtrace", python["PYTEST_ADDOPTS"]) + ruby := (&Testdrive{language: "ruby"}).environment("/session/Gemfile", "http://127.0.0.1:1234", "session") + require.True(t, strings.HasPrefix(ruby["RUBYOPT"], "-W0 ")) + require.Equal(t, "/session/gems", ruby["BUNDLE_PATH"]) +} + +func TestCypressWrapperUsesExplicitConfigWithoutEditingIt(t *testing.T) { + root := t.TempDir() + session := t.TempDir() + config := `module.exports={e2e:{supportFile:false}}` + requireWriteFile(t, filepath.Join(root, "custom.cjs"), config) + args, err := prepareCypress(root, session, "/tracer/ci/init.js", []string{"run", "--config-file=custom.cjs", "--browser", "chrome"}) + require.NoError(t, err) + require.Equal(t, []string{"run", "--browser", "chrome", "--config-file", filepath.Join(session, "cypress.config.cjs")}, args) + contents, err := os.ReadFile(filepath.Join(root, "custom.cjs")) + require.NoError(t, err) + require.Equal(t, config, string(contents)) + wrapper, err := os.ReadFile(filepath.Join(session, "cypress.config.cjs")) + require.NoError(t, err) + require.Contains(t, string(wrapper), filepath.Join(root, "custom.cjs")) + _, err = prepareCypress(root, session, "/tracer/ci/init.js", []string{"--config-file"}) + require.ErrorContains(t, err, "requires a path") +} diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go new file mode 100644 index 00000000..04a2461d --- /dev/null +++ b/internal/testdrive/report.go @@ -0,0 +1,841 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "errors" + "fmt" + "html/template" + "net/url" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "github.com/DataDog/ddtest/internal/testdrive/intake" +) + +const reportFilename = "report.html" + +type reportCard struct { + Title string + Count int + Context string + Tests []reportTest + Coverages []reportCoverage +} + +type reportTest struct { + Label string + Name string + Suite string + SourceFile string + Status string + Tone string + Duration string + Attempts []reportAttempt + CoverageLevel string + CoveredFiles []string + Source reportSource +} + +type reportAttempt struct { + Number int + Status string + Tone string + Duration string + Kind string + ErrorType string + ErrorMessage string + ErrorStack string +} + +type reportCoverage struct { + Name string + Level string + FileCount int + Files []string + SourceFile string + Source reportSource +} + +type reportSource struct { + Start int + End int + Lines []reportSourceLine + Error string +} + +type reportSourceLine struct { + Number int + Code template.HTML +} + +type reportSuite struct { + Name string + Status string + Tone string + Duration string + TestCount int + CoveredCount int + ShowCoverage bool + CoveredFiles []string + Tests []reportSuiteTest +} + +type reportSuiteTest struct { + Name string + Status string + Tone string + Duration string +} + +type reportFact struct { + Label string + Value string + Tone string +} + +type reportArtifact struct { + Title string + Href string +} + +type reportModel struct { + Headline string + Summary string + Cards []reportCard + Facts []reportFact + Artifacts []reportArtifact + Suites []reportSuite + Tests []reportTest +} + +type reportRuntime struct{ Framework, Tracer string } + +func writeReport(repositoryRoot, sessionDirectory string, findings intake.Findings, commandFailed bool, runtime ...reportRuntime) (string, error) { + path := filepath.Join(sessionDirectory, reportFilename) + file, err := os.Create(path) + if err != nil { + return "", fmt.Errorf("create testdrive report: %w", err) + } + + model := buildReport(repositoryRoot, findings, commandFailed, runtime...) + executeErr := testdriveReport.Execute(file, model) + closeErr := file.Close() + if err := errors.Join(executeErr, closeErr); err != nil { + return "", fmt.Errorf("write testdrive report: %w", err) + } + return path, nil +} + +func buildReport(repositoryRoot string, findings intake.Findings, commandFailed bool, runtime ...reportRuntime) reportModel { + info := reportRuntime{Framework: "Test command", Tracer: "Isolated installation"} + if len(runtime) > 0 { + info = runtime[0] + } + showTestCoverage := findings.CoverageLevel == "test" + showSuiteCoverage := findings.CoverageLevel == "suite" + model := reportModel{ + Headline: "Test Optimization is ready.", + Summary: "No findings.", + Facts: []reportFact{ + {Label: "Test events", Value: fmt.Sprintf("%d", findings.TestEventCount), Tone: factTone(findings.TestEventCount > 0)}, + {Label: "Tests with coverage", Value: fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount), Tone: factTone(findings.TestCount > 0 && findings.CoveredTestCount == findings.TestCount)}, + {Label: info.Framework, Value: passedFailed(!commandFailed), Tone: factTone(!commandFailed)}, + {Label: "Tracer", Value: info.Tracer + " · isolated", Tone: "good"}, + }, + Artifacts: []reportArtifact{ + {Title: "JSON traffic", Href: "intake/"}, + {Title: "Test output", Href: testOutputFilename}, + }, + Tests: reportTests(repositoryRoot, findings.Tests, showTestCoverage), + } + model.Suites = reportSuites(findings.Tests, showSuiteCoverage) + if findings.CoveredTestCount == 0 { + model.Facts[1].Value = "Not reported" + } + + if findings.TestEventCount == 0 { + model.Headline = "No test events received." + model.Summary = "Check the instrumentation setup." + } + + if len(findings.FailedTests) > 0 { + model.Cards = append(model.Cards, reportCard{ + Title: "Any tests failed?", Count: len(findings.FailedTests), + Tests: reportTests(repositoryRoot, findings.FailedTests, showTestCoverage), + }) + } + if len(findings.FlakyTests) > 0 { + model.Cards = append(model.Cards, reportCard{ + Title: "Any flaky tests?", Count: len(findings.FlakyTests), + Tests: reportTests(repositoryRoot, findings.FlakyTests, showTestCoverage), + }) + } + if len(findings.SlowTests) > 0 { + model.Cards = append(model.Cards, reportCard{ + Title: "Any tests slower than the others?", Count: len(findings.SlowTests), + Context: "Median test time · " + formatDuration(findings.TestDurationMedian), + Tests: reportTests(repositoryRoot, findings.SlowTests, showTestCoverage), + }) + } + if len(findings.BroadCoverage) > 0 { + model.Cards = append(model.Cards, reportCard{ + Title: "Any unusually broad test coverage?", Count: len(findings.BroadCoverage), + Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), + Coverages: reportCoverages(repositoryRoot, findings.BroadCoverage), + }) + } + if len(model.Cards) > 0 && findings.TestEventCount > 0 { + model.Summary = fmt.Sprintf("%d findings.", len(model.Cards)) + } + return model +} + +func reportTests(repositoryRoot string, findings []intake.TestFinding, showCoverage bool) []reportTest { + tests := make([]reportTest, 0, len(findings)) + for _, finding := range findings { + label := finding.Name + if finding.Suite != "" { + label = finding.Suite + " › " + finding.Name + } + status, tone := testDisplayStatus(finding) + test := reportTest{ + Label: label, Name: finding.Name, Suite: finding.Suite, + SourceFile: finding.SourceFile, Status: status, Tone: tone, + Duration: formatDuration(findingDuration(finding)), + Attempts: make([]reportAttempt, 0, len(finding.Attempts)), + Source: readSource(repositoryRoot, finding.SourceFile, finding.SourceStart, finding.SourceEnd), + } + if showCoverage && finding.CoverageLevel == "test" { + test.CoverageLevel = finding.CoverageLevel + test.CoveredFiles = finding.CoveredFiles + } + for attemptIndex, attempt := range finding.Attempts { + kind := "Initial run" + if attempt.Retry { + kind = "Retry" + if attempt.RetryReason != "" { + kind += " · " + strings.ReplaceAll(attempt.RetryReason, "_", " ") + } + } + test.Attempts = append(test.Attempts, reportAttempt{ + Number: attemptIndex + 1, Status: displayStatus(attempt.Status), Tone: attemptTone(attempt.Status), + Duration: formatDuration(attempt.Duration), Kind: kind, + ErrorType: attempt.ErrorType, ErrorMessage: attempt.ErrorMessage, ErrorStack: attempt.ErrorStack, + }) + } + tests = append(tests, test) + } + return tests +} + +func reportCoverages(repositoryRoot string, findings []intake.CoverageFinding) []reportCoverage { + coverages := make([]reportCoverage, 0, len(findings)) + for _, finding := range findings { + files := slices.Clone(finding.Files) + slices.Sort(files) + coverage := reportCoverage{ + Name: finding.Name, Level: finding.Level, FileCount: finding.FileCount, + Files: files, SourceFile: finding.SourceFile, + } + if finding.Level == "test" { + coverage.Source = readSource(repositoryRoot, finding.SourceFile, finding.SourceStart, finding.SourceEnd) + } + coverages = append(coverages, coverage) + } + return coverages +} + +func reportSuites(tests []intake.TestFinding, showCoverage bool) []reportSuite { + byName := make(map[string]*reportSuite) + durations := make(map[string]time.Duration) + for _, test := range tests { + name := test.Suite + if name == "" { + name = "Unknown suite" + } + suite, found := byName[name] + if !found { + suite = &reportSuite{Name: name, Status: "Passed", Tone: "good", ShowCoverage: showCoverage} + byName[name] = suite + } + status, tone := testDisplayStatus(test) + if status == "Fail" { + suite.Status = "Failed" + suite.Tone = "attention" + } else if status == "Flaky" && suite.Status != "Failed" { + suite.Status = "Flaky" + suite.Tone = "attention" + } + suite.TestCount++ + durations[name] += findingDuration(test) + if showCoverage && test.CoverageLevel == "suite" { + suite.CoveredCount++ + suite.CoveredFiles = appendUniqueStrings(suite.CoveredFiles, test.CoveredFiles...) + } + suite.Tests = append(suite.Tests, reportSuiteTest{ + Name: test.Name, Status: status, Tone: tone, + Duration: formatDuration(findingDuration(test)), + }) + } + + suites := make([]reportSuite, 0, len(byName)) + for _, suite := range byName { + suite.Duration = formatDuration(durations[suite.Name]) + suites = append(suites, *suite) + } + sort.Slice(suites, func(i, j int) bool { return suites[i].Name < suites[j].Name }) + return suites +} + +func testDisplayStatus(test intake.TestFinding) (string, string) { + status := test.Status + sawPass := status == "pass" + sawFailure := status == "fail" + for _, attempt := range test.Attempts { + sawPass = sawPass || attempt.Status == "pass" + sawFailure = sawFailure || attempt.Status == "fail" + } + if sawPass && sawFailure { + return "Flaky", "attention" + } + if status == "" && len(test.Attempts) > 0 { + status = test.Attempts[len(test.Attempts)-1].Status + } + return displayStatus(status), attemptTone(status) +} + +func findingDuration(test intake.TestFinding) time.Duration { + if test.Duration != 0 || len(test.Attempts) == 0 { + return test.Duration + } + return test.Attempts[0].Duration +} + +func appendUniqueStrings(values []string, additions ...string) []string { + for _, addition := range additions { + if slices.Contains(values, addition) { + continue + } + values = append(values, addition) + } + slices.Sort(values) + return values +} + +func readSource(repositoryRoot, sourceFile string, sourceStart, sourceEnd int) reportSource { + if sourceFile == "" { + return reportSource{Error: "Source file not reported."} + } + if sourceStart < 1 { + return reportSource{Error: "Source line not reported."} + } + path := sourceFile + if !filepath.IsAbs(path) { + path = filepath.Join(repositoryRoot, filepath.FromSlash(path)) + } + contents, err := os.ReadFile(path) + if err != nil { + return reportSource{Error: "Source could not be read: " + err.Error()} + } + lines := strings.Split(strings.ReplaceAll(string(contents), "\r\n", "\n"), "\n") + if sourceStart > len(lines) { + return reportSource{Error: fmt.Sprintf("Source line %d is outside %s.", sourceStart, sourceFile)} + } + if sourceEnd < sourceStart { + if filepath.Ext(path) == ".py" || filepath.Ext(path) == ".rb" { + sourceEnd = min(sourceStart+11, len(lines)) + } else { + sourceEnd = inferJavaScriptTestEnd(lines, sourceStart) + } + } + sourceEnd = min(sourceEnd, len(lines)) + source := reportSource{Start: sourceStart, End: sourceEnd} + inBlockComment := false + for lineIndex := sourceStart - 1; lineIndex < sourceEnd; lineIndex++ { + code := highlightJavaScriptLine(lines[lineIndex], &inBlockComment) + if filepath.Ext(path) == ".py" || filepath.Ext(path) == ".rb" { + code = template.HTML(template.HTMLEscapeString(lines[lineIndex])) + } + source.Lines = append(source.Lines, reportSourceLine{ + Number: lineIndex + 1, + Code: code, + }) + } + return source +} + +func inferJavaScriptTestEnd(lines []string, sourceStart int) int { + parentheses := 0 + sawParenthesis := false + inBlockComment := false + var quote byte + escaped := false + for lineIndex := sourceStart - 1; lineIndex < len(lines); lineIndex++ { + line := lines[lineIndex] + for characterIndex := 0; characterIndex < len(line); characterIndex++ { + character := line[characterIndex] + if inBlockComment { + if character == '*' && characterIndex+1 < len(line) && line[characterIndex+1] == '/' { + inBlockComment = false + characterIndex++ + } + continue + } + if quote != 0 { + if escaped { + escaped = false + continue + } + if character == '\\' { + escaped = true + continue + } + if character == quote { + quote = 0 + } + continue + } + if character == '/' && characterIndex+1 < len(line) { + switch line[characterIndex+1] { + case '/': + characterIndex = len(line) + continue + case '*': + inBlockComment = true + characterIndex++ + continue + } + } + if character == '\'' || character == '"' || character == '`' { + quote = character + continue + } + switch character { + case '(': + parentheses++ + sawParenthesis = true + case ')': + parentheses-- + if sawParenthesis && parentheses == 0 && onlyStatementEnd(line[characterIndex+1:]) { + return lineIndex + 1 + } + } + } + } + return min(sourceStart+19, len(lines)) +} + +func onlyStatementEnd(value string) bool { + value = strings.TrimSpace(value) + return value == "" || value == ";" || strings.HasPrefix(value, "//") +} + +func highlightJavaScriptLine(line string, inBlockComment *bool) template.HTML { + var highlighted strings.Builder + for index := 0; index < len(line); { + if *inBlockComment { + end := strings.Index(line[index:], "*/") + if end < 0 { + writeToken(&highlighted, "comment", line[index:]) + break + } + end += index + 2 + writeToken(&highlighted, "comment", line[index:end]) + *inBlockComment = false + index = end + continue + } + if strings.HasPrefix(line[index:], "//") { + writeToken(&highlighted, "comment", line[index:]) + break + } + if strings.HasPrefix(line[index:], "/*") { + end := strings.Index(line[index+2:], "*/") + if end < 0 { + writeToken(&highlighted, "comment", line[index:]) + *inBlockComment = true + break + } + end += index + 4 + writeToken(&highlighted, "comment", line[index:end]) + index = end + continue + } + if line[index] == '\'' || line[index] == '"' || line[index] == '`' { + end := stringEnd(line, index) + writeToken(&highlighted, "string", line[index:end]) + index = end + continue + } + if isIdentifierStart(line[index]) { + end := index + 1 + for end < len(line) && isIdentifierPart(line[end]) { + end++ + } + word := line[index:end] + class := "" + if javascriptKeywords[word] { + class = "keyword" + } else if javascriptLiterals[word] { + class = "literal" + } + writeToken(&highlighted, class, word) + index = end + continue + } + if line[index] >= '0' && line[index] <= '9' { + end := index + 1 + for end < len(line) && ((line[end] >= '0' && line[end] <= '9') || line[end] == '.') { + end++ + } + writeToken(&highlighted, "number", line[index:end]) + index = end + continue + } + writeToken(&highlighted, "", line[index:index+1]) + index++ + } + return template.HTML(highlighted.String()) +} + +func stringEnd(line string, start int) int { + quote := line[start] + escaped := false + for index := start + 1; index < len(line); index++ { + if escaped { + escaped = false + continue + } + if line[index] == '\\' { + escaped = true + continue + } + if line[index] == quote { + return index + 1 + } + } + return len(line) +} + +func writeToken(output *strings.Builder, class, value string) { + escaped := template.HTMLEscapeString(value) + if class == "" { + output.WriteString(escaped) + return + } + output.WriteString(``) + output.WriteString(escaped) + output.WriteString(``) +} + +func isIdentifierStart(character byte) bool { + return character == '_' || character == '$' || character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' +} + +func isIdentifierPart(character byte) bool { + return isIdentifierStart(character) || character >= '0' && character <= '9' +} + +var javascriptKeywords = map[string]bool{ + "async": true, "await": true, "break": true, "case": true, "catch": true, "class": true, + "const": true, "continue": true, "default": true, "delete": true, "do": true, "else": true, + "export": true, "extends": true, "finally": true, "for": true, "from": true, "function": true, + "if": true, "import": true, "in": true, "instanceof": true, "let": true, "new": true, + "of": true, "return": true, "switch": true, "throw": true, "try": true, "typeof": true, + "var": true, "void": true, "while": true, "with": true, "yield": true, +} + +var javascriptLiterals = map[string]bool{ + "false": true, "null": true, "true": true, "undefined": true, +} + +func passedFailed(passed bool) string { + if passed { + return "Passed" + } + return "Failed" +} + +func factTone(good bool) string { + if good { + return "good" + } + return "attention" +} + +func displayStatus(status string) string { + if status == "" { + return "Unknown" + } + return strings.ToUpper(status[:1]) + status[1:] +} + +func attemptTone(status string) string { + if status == "pass" { + return "good" + } + return "attention" +} + +func formatDuration(duration time.Duration) string { + if duration < time.Millisecond { + return duration.Round(time.Microsecond).String() + } + return duration.Round(time.Millisecond).String() +} + +func fileURL(path string) (string, error) { + absolutePath, err := filepath.Abs(path) + if err != nil { + return "", err + } + return (&url.URL{Scheme: "file", Path: absolutePath}).String(), nil +} + +func terminalLink(target string) string { + return "\x1b]8;;" + target + "\x1b\\" + target + "\x1b]8;;\x1b\\" +} + +func plural(count int, singular, plural string) string { + if count == 1 { + return singular + } + return plural +} + +var testdriveReport = template.Must(template.New("testdrive-report").Funcs(template.FuncMap{"plural": plural}).Parse(` +{{define "attempts"}} +
+ {{range .Attempts}}
+
+ {{.Status}} + Run {{.Number}} · {{.Kind}} + {{.Duration}} +
+ {{if .ErrorMessage}}

{{if .ErrorType}}{{.ErrorType}}: {{end}}{{.ErrorMessage}}

{{end}} + {{if .ErrorStack}}
Stack trace
{{.ErrorStack}}
{{end}} +
{{end}} +
+{{end}} +{{define "source"}} + {{if .Lines}}
+
Source · lines {{.Start}}–{{.End}}
+ 
{{range .Lines}}{{.Number}}{{.Code}}{{end}}
+
{{else if .Error}}

{{.Error}}

{{end}} +{{end}} +{{define "covered-files"}} + {{if .}}
+
    {{range .}}
  • {{.}}
    • {{end}}
+
+
{{end}} +{{end}} +{{define "test-detail"}} +
+ {{if .SourceFile}}

{{.SourceFile}}

{{end}} + {{template "attempts" .}} + {{if .CoverageLevel}}
Coverage · {{.CoverageLevel}} level · {{len .CoveredFiles}} {{plural (len .CoveredFiles) "file" "files"}}{{template "covered-files" .CoveredFiles}}
{{end}} + {{template "source" .Source}} +
+{{end}} + + + + + + ddtest · Test Optimization report + + + +
+

ddtest · local testdrive

+

{{.Headline}}

+

{{.Summary}}

+ + + +
+ {{if .Cards}}
+ {{range .Cards}}
+

{{.Title}}

{{if .Context}}{{.Context}}{{end}}{{.Count}}
+
+ {{range .Tests}}
+ {{.Label}}{{.Status}} · {{.Duration}} + {{template "test-detail" .}} +
{{end}} + {{range .Coverages}}
+ {{.Name}}{{.FileCount}} {{plural .FileCount "file" "files"}} · {{.Level}} level +
+ {{if .SourceFile}}

{{.SourceFile}}

{{end}} + {{template "covered-files" .Files}} + {{template "source" .Source}} +
+
{{end}} +
+
{{end}} +
{{end}} + +
+

Run details

+
{{range .Facts}}
{{.Label}}
{{.Value}}
{{end}}
+
{{range .Artifacts}}{{.Title}} ↗{{end}}
+
+
+ + + + +
+ + + +`)) diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go new file mode 100644 index 00000000..9f002996 --- /dev/null +++ b/internal/testdrive/report_test.go @@ -0,0 +1,269 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package testdrive + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/DataDog/ddtest/internal/testdrive/intake" +) + +func TestReportKeepsPythonAndRubySourceEscaped(t *testing.T) { + for _, extension := range []string{".py", ".rb"} { + t.Run(extension, func(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "test"+extension), []byte("value = ''\nassert value\n"), 0644); err != nil { + t.Fatal(err) + } + source := readSource(root, "test"+extension, 1, 0) + if source.Error != "" || len(source.Lines) < 2 { + t.Fatalf("source not rendered: %+v", source) + } + if !strings.Contains(string(source.Lines[0].Code), "<script>") || strings.Contains(string(source.Lines[0].Code), "