diff --git a/internal/testdrive/intake/coverage_test.go b/internal/testdrive/intake/coverage_test.go index 66cbe0d9..4a119a36 100644 --- a/internal/testdrive/intake/coverage_test.go +++ b/internal/testdrive/intake/coverage_test.go @@ -160,7 +160,7 @@ func TestCoverageTrackingKeepsValidPayloads(t *testing.T) { require.Equal(t, 2, emptyEntries) } -func TestFindingsListsEveryTestWithSuiteCoverage(t *testing.T) { +func TestFindingsKeepsSuiteCoverageOffTests(t *testing.T) { const sessionID = 10 events := msgp.AppendMapHeader(nil, 1) events = msgp.AppendString(events, "events") @@ -174,9 +174,10 @@ func TestFindingsListsEveryTestWithSuiteCoverage(t *testing.T) { require.Equal(t, 2, findings.TestCount) require.Len(t, findings.Tests, 2) for _, test := range findings.Tests { - require.Equal(t, "suite", test.CoverageLevel) - require.Equal(t, []string{"src/one.js", "src/two.js"}, test.CoveredFiles) + require.Empty(t, test.CoverageLevel) + require.Empty(t, test.CoveredFiles) } + require.Equal(t, []SuiteCoverage{{Suite: "one.test.js", Files: []string{"src/one.js", "src/two.js"}, CoveredTests: 2}}, findings.SuiteCoverages) } func serverWithCoverage(t *testing.T, events []byte, coverageEntries ...[]byte) *Server { diff --git a/internal/testdrive/intake/findings.go b/internal/testdrive/intake/findings.go index 1a5a7252..56256dd8 100644 --- a/internal/testdrive/intake/findings.go +++ b/internal/testdrive/intake/findings.go @@ -19,7 +19,8 @@ import ( ) const ( - minimumSlowDuration = 5 * time.Second + minimumSlowTestDuration = time.Second + minimumSlowSuiteDuration = 5 * time.Second minimumBroadCoverageFiles = 5 ) @@ -61,6 +62,21 @@ type CoverageFact struct { Files []string } +// SuiteCoverage holds files covered by a suite, separate from individual tests. +type SuiteCoverage struct { + Module string + Suite string + Files []string + CoveredTests int +} + +// SlowSuite describes a suite whose combined test duration is unusually high. +type SlowSuite struct { + Module string + Suite string + Duration time.Duration +} + // Facts contains the facts shown in the testdrive report. type Facts struct { ConfigurationErrors []string @@ -69,12 +85,15 @@ type Facts struct { TestEventCount int CoveredTestCount int TestDurationMedian time.Duration + SuiteDurationMedian time.Duration CoveredFilesMedian int CoverageLevel string + SuiteCoverages []SuiteCoverage Tests []Test FailedTests []Test FlakyTests []Test SlowTests []Test + SlowSuites []SlowSuite BroadCoverage []CoverageFact } @@ -91,6 +110,10 @@ func (s *Server) Facts() (Facts, error) { findings := Facts{TestEventCount: len(tests), CoverageLevel: coverageLevel(coverages), EmptyCoverageEntryCount: emptyEntries} findings.Tests, findings.FailedTests, findings.FlakyTests, findings.SlowTests, findings.TestDurationMedian = analyzeTests(tests, coverages, findings.CoverageLevel) + if findings.CoverageLevel == "suite" { + findings.SuiteCoverages = suiteCoverages(tests, coverages) + } + findings.SlowSuites, findings.SuiteDurationMedian = slowSuites(findings.Tests) findings.TestCount = len(findings.Tests) findings.CoveredTestCount = uniqueCoveredTestCount(tests, coverages) findings.BroadCoverage, findings.CoveredFilesMedian = analyzeCoverage(tests, coverages, findings.CoverageLevel) @@ -99,7 +122,10 @@ func (s *Server) Facts() (Facts, error) { } func analyzeTests(tests []testReference, coverages []coverageReference, level string) ([]Test, []Test, []Test, []Test, time.Duration) { - filesByTest := coverageFilesByTest(tests, coverages, level) + var filesByTest map[string][]string + if level == "test" { + filesByTest = coverageFilesByTest(tests, coverages) + } testsByName := make(map[string][]testReference) order := make([]string, 0) for _, test := range tests { @@ -117,7 +143,7 @@ func analyzeTests(tests []testReference, coverages []coverageReference, level st attempts := testsByName[key] finding := testFromReference(attempts[0]) if files, covered := filesByTest[key]; covered { - finding.CoverageLevel = level + finding.CoverageLevel = "test" finding.CoveredFiles = files } finding.Attempts = make([]TestRun, 0, len(attempts)) @@ -167,47 +193,86 @@ func analyzeTests(tests []testReference, coverages []coverageReference, level st return all, failed, flaky, slow, median } -func coverageFilesByTest(tests []testReference, coverages []coverageReference, level string) map[string][]string { +func coverageFilesByTest(tests []testReference, coverages []coverageReference) map[string][]string { testsBySpan := make(map[uint64]string, len(tests)) for _, test := range tests { testsBySpan[test.spanID] = testIdentity(test) } - testFiles := make(map[string][]string) - suiteFiles := make(map[suiteReference][]string) + filesByTest := make(map[string]map[string]struct{}) for _, coverage := range coverages { - if level == "test" && coverage.spanID != 0 { - if identity := testsBySpan[coverage.spanID]; identity != "" { - testFiles[identity] = appendUnique(testFiles[identity], coverage.files...) - } - } - if level != "suite" || coverage.spanID != 0 { + if coverage.spanID == 0 { continue } - key := suiteReference{sessionID: coverage.sessionID, suiteID: coverage.suiteID} - suiteFiles[key] = appendUnique(suiteFiles[key], coverage.files...) - } - - if level == "suite" { - for _, test := range tests { - key := suiteReference{sessionID: test.sessionID, suiteID: test.suiteID} - if files, covered := suiteFiles[key]; covered { - identity := testIdentity(test) - testFiles[identity] = appendUnique(testFiles[identity], files...) + if identity := testsBySpan[coverage.spanID]; identity != "" { + if filesByTest[identity] == nil { + filesByTest[identity] = make(map[string]struct{}) + } + for _, file := range coverage.files { + filesByTest[identity][file] = struct{}{} } } } + testFiles := make(map[string][]string, len(filesByTest)) + for identity, files := range filesByTest { + testFiles[identity] = sortedCoverageFiles(files) + } return testFiles } -func appendUnique(values []string, additions ...string) []string { - for _, addition := range additions { - if slices.Contains(values, addition) { +func suiteCoverages(tests []testReference, coverages []coverageReference) []SuiteCoverage { + testsBySuite := make(map[suiteReference]testReference) + for _, test := range tests { + testsBySuite[suiteReference{sessionID: test.sessionID, suiteID: test.suiteID}] = test + } + byName := make(map[string]*SuiteCoverage) + filesByName := make(map[string]map[string]struct{}) + coveredSuites := make(map[suiteReference]struct{}) + for _, coverage := range coverages { + if coverage.spanID != 0 { + continue + } + suiteID := suiteReference{sessionID: coverage.sessionID, suiteID: coverage.suiteID} + test, found := testsBySuite[suiteID] + if !found { continue } - values = append(values, addition) + coveredSuites[suiteID] = struct{}{} + key := test.module + "\x00" + test.suite + if _, found := byName[key]; !found { + byName[key] = &SuiteCoverage{Module: test.module, Suite: test.suite} + filesByName[key] = make(map[string]struct{}) + } + for _, file := range coverage.files { + filesByName[key][file] = struct{}{} + } } - slices.Sort(values) - return values + coveredTests := make(map[string]map[string]struct{}) + for _, test := range tests { + if _, found := coveredSuites[suiteReference{sessionID: test.sessionID, suiteID: test.suiteID}]; !found { + continue + } + key := test.module + "\x00" + test.suite + if coveredTests[key] == nil { + coveredTests[key] = make(map[string]struct{}) + } + coveredTests[key][testIdentity(test)] = struct{}{} + } + result := make([]SuiteCoverage, 0, len(byName)) + for key, finding := range byName { + finding.Files = sortedCoverageFiles(filesByName[key]) + finding.CoveredTests = len(coveredTests[key]) + result = append(result, *finding) + } + return result +} + +func sortedCoverageFiles(files map[string]struct{}) []string { + result := make([]string, 0, len(files)) + for file := range files { + result = append(result, file) + } + slices.Sort(result) + return result } func slowTests(tests []Test) ([]Test, time.Duration) { @@ -218,7 +283,7 @@ func slowTests(tests []Test) ([]Test, time.Duration) { slow := make([]Test, 0) for _, test := range tests { - if test.Duration > minimumSlowDuration && test.Duration >= median*5 { + if test.Duration > minimumSlowTestDuration && test.Duration >= median*5 { slow = append(slow, test) } } @@ -226,6 +291,41 @@ func slowTests(tests []Test) ([]Test, time.Duration) { return slow, median } +func slowSuites(tests []Test) ([]SlowSuite, time.Duration) { + bySuite := make(map[string]SlowSuite) + for _, test := range tests { + key := test.Module + "\x00" + test.Suite + suite := bySuite[key] + suite.Module, suite.Suite = test.Module, test.Suite + suite.Duration += test.Duration + bySuite[key] = suite + } + durations := make([]time.Duration, 0, len(bySuite)) + for _, suite := range bySuite { + durations = append(durations, suite.Duration) + } + median := medianDurations(durations) + var slow []SlowSuite + if len(bySuite) < 2 { + return slow, median + } + for _, suite := range bySuite { + if suite.Duration > minimumSlowSuiteDuration && suite.Duration >= median*5 { + slow = append(slow, suite) + } + } + sort.Slice(slow, func(i, j int) bool { + if slow[i].Duration != slow[j].Duration { + return slow[i].Duration > slow[j].Duration + } + if slow[i].Module != slow[j].Module { + return slow[i].Module < slow[j].Module + } + return slow[i].Suite < slow[j].Suite + }) + return slow, median +} + func medianTestDuration(tests []Test) time.Duration { if len(tests) == 0 { return 0 @@ -234,6 +334,13 @@ func medianTestDuration(tests []Test) time.Duration { for _, test := range tests { durations = append(durations, test.Duration) } + return medianDurations(durations) +} + +func medianDurations(durations []time.Duration) time.Duration { + if len(durations) == 0 { + return 0 + } sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) middle := len(durations) / 2 if len(durations)%2 == 1 { @@ -291,6 +398,9 @@ func analyzeCoverage(tests []testReference, coverages []coverageReference, level key = fmt.Sprintf("%d/%d", coverage.sessionID, coverage.suiteID) finding.Level = "suite" finding.Name = test.suite + if test.module != "" { + finding.Name = test.module + " › " + test.suite + } finding.SourceFile = test.sourceFile } if current, found := findingsByName[key]; !found || finding.FileCount > current.FileCount { diff --git a/internal/testdrive/intake/findings_test.go b/internal/testdrive/intake/findings_test.go index 45177c60..0137213d 100644 --- a/internal/testdrive/intake/findings_test.go +++ b/internal/testdrive/intake/findings_test.go @@ -54,8 +54,8 @@ func TestAnalyzeTestsSlowThresholds(t *testing.T) { wantSlow bool }{ {name: "fast outlier", median: 10 * time.Millisecond, duration: 400 * time.Millisecond}, - {name: "exactly five seconds", median: time.Second, duration: 5 * time.Second}, - {name: "above five seconds", median: time.Second, duration: 5*time.Second + time.Nanosecond, wantSlow: true}, + {name: "exactly one second", median: 100 * time.Millisecond, duration: time.Second}, + {name: "above one second", median: 100 * time.Millisecond, duration: time.Second + time.Nanosecond, wantSlow: true}, {name: "below five times median", median: 2 * time.Second, duration: 10*time.Second - time.Nanosecond}, {name: "exactly five times median", median: 2 * time.Second, duration: 10 * time.Second, wantSlow: true}, {name: "above five times median", median: 2 * time.Second, duration: 11 * time.Second, wantSlow: true}, @@ -112,12 +112,30 @@ func TestAddCoverageToTestsUsesActiveCoverageLevel(t *testing.T) { t.Run("suite", func(t *testing.T) { findings, _, _, _, _ := analyzeTests(tests, coverages, "suite") for _, finding := range findings { - require.Equal(t, "suite", finding.CoverageLevel) - require.Equal(t, []string{"shared.js"}, finding.CoveredFiles) + require.Empty(t, finding.CoverageLevel) + require.Empty(t, finding.CoveredFiles) } + require.Equal(t, []SuiteCoverage{{Suite: "one.test.js", Files: []string{"shared.js"}, CoveredTests: 2}}, suiteCoverages(tests, coverages)) }) } +func TestSuiteCoveragesKeepsModuleIdentityAndPartialCounts(t *testing.T) { + tests := []testReference{ + {sessionID: 1, suiteID: 10, spanID: 1, module: "first", suite: "shared", name: "one"}, + {sessionID: 1, suiteID: 10, spanID: 2, module: "first", suite: "shared", name: "two"}, + {sessionID: 2, suiteID: 20, spanID: 3, module: "first", suite: "shared", name: "uncovered"}, + {sessionID: 3, suiteID: 30, spanID: 4, module: "second", suite: "shared", name: "other"}, + } + coverages := []coverageReference{ + {testReference: testReference{sessionID: 1, suiteID: 10}, files: []string{"first.js"}}, + {testReference: testReference{sessionID: 3, suiteID: 30}, files: []string{"second.js"}}, + } + require.ElementsMatch(t, []SuiteCoverage{ + {Module: "first", Suite: "shared", Files: []string{"first.js"}, CoveredTests: 2}, + {Module: "second", Suite: "shared", Files: []string{"second.js"}, CoveredTests: 1}, + }, suiteCoverages(tests, coverages)) +} + func TestAnalyzeCoverageUsesActiveCoverageLevel(t *testing.T) { tests := []testReference{ {sessionID: 1, suiteID: 10, spanID: 100, name: "narrow", suite: "one.test.js"}, @@ -201,14 +219,25 @@ func TestMedianHelpers(t *testing.T) { })) } -func TestCoverageLevelAndAppendUnique(t *testing.T) { +func TestCoverageLevel(t *testing.T) { require.Empty(t, coverageLevel(nil)) require.Equal(t, "suite", coverageLevel([]coverageReference{{testReference: testReference{suiteID: 1}}})) require.Equal(t, "test", coverageLevel([]coverageReference{ {testReference: testReference{suiteID: 1}}, {testReference: testReference{spanID: 2}}, })) - require.Equal(t, []string{"a.js", "b.js"}, appendUnique([]string{"b.js"}, "a.js", "b.js")) +} + +func TestCoverageFilesMergeDuplicatesAndSortOnce(t *testing.T) { + tests := []testReference{{sessionID: 1, suiteID: 10, spanID: 100, name: "one", suite: "suite"}} + coverages := []coverageReference{ + {testReference: testReference{spanID: 100}, files: []string{"b.js", "a.js", "b.js"}}, + {testReference: testReference{spanID: 100}, files: []string{"c.js", "a.js"}}, + {testReference: testReference{sessionID: 1, suiteID: 10}, files: []string{"y.js", "x.js", "y.js"}}, + {testReference: testReference{sessionID: 1, suiteID: 10}, files: []string{"z.js", "x.js"}}, + } + require.Equal(t, map[string][]string{testIdentity(tests[0]): {"a.js", "b.js", "c.js"}}, coverageFilesByTest(tests, coverages)) + require.Equal(t, []SuiteCoverage{{Suite: "suite", Files: []string{"x.js", "y.js", "z.js"}, CoveredTests: 1}}, suiteCoverages(tests, coverages)) } func TestFindingsIncludeConfigurationErrorsAcrossEventLevels(t *testing.T) { @@ -305,11 +334,19 @@ func TestFindingsPreservesCoverageInEveryCategory(t *testing.T) { require.Len(t, findings.SlowTests, 1) for _, category := range [][]Test{findings.Tests, findings.FailedTests, findings.FlakyTests, findings.SlowTests} { for _, finding := range category { - require.Equal(t, level, finding.CoverageLevel) - require.Equal(t, []string{"covered.js"}, finding.CoveredFiles) + if level == "test" { + require.Equal(t, "test", finding.CoverageLevel) + require.Equal(t, []string{"covered.js"}, finding.CoveredFiles) + } else { + require.Empty(t, finding.CoverageLevel) + require.Empty(t, finding.CoveredFiles) + } require.Contains(t, findings.Tests, finding) } } + if level == "suite" { + require.Equal(t, []SuiteCoverage{{Files: []string{"covered.js"}, CoveredTests: 4}}, findings.SuiteCoverages) + } }) } } @@ -340,3 +377,62 @@ func TestAnalyzeTestsDoesNotUseSourceLocationAsIdentity(t *testing.T) { require.Len(t, flaky[0].Attempts, 2) require.Empty(t, failed) } + +func TestSlowSuiteThresholds(t *testing.T) { + for _, tc := range []struct { + name string + median, duration time.Duration + want bool + }{ + {"below floor", time.Millisecond, 4 * time.Second, false}, + {"at floor", time.Second, 5 * time.Second, false}, + {"above floor", time.Second, 5*time.Second + time.Nanosecond, true}, + {"below ratio", 2 * time.Second, 10*time.Second - time.Nanosecond, false}, + {"at ratio", 2 * time.Second, 10 * time.Second, true}, + {"uniform", 6 * time.Second, 6 * time.Second, false}, + } { + t.Run(tc.name, func(t *testing.T) { + slow, median := slowSuites([]Test{ + {Suite: "one", Duration: tc.median}, {Suite: "two", Duration: tc.median}, + {Suite: "candidate", Duration: tc.duration}, + }) + require.Equal(t, tc.median, median) + if tc.want { + require.Equal(t, []SlowSuite{{Suite: "candidate", Duration: tc.duration}}, slow) + } else { + require.Empty(t, slow) + } + }) + } + for _, tests := range [][]Test{nil, {{Suite: "only", Duration: time.Minute}}} { + slow, _ := slowSuites(tests) + require.Empty(t, slow) + } +} + +func TestFactsFindSlowSuitesWithoutCountingRetriesTwice(t *testing.T) { + var events []any + for i, tc := range []struct { + module, suite, name string + duration time.Duration + retry bool + }{ + {"one", "shared", "baseline", time.Second, false}, + {"two", "shared", "baseline", time.Second, false}, + {"three", "shared", "first", 4 * time.Second, false}, + {"three", "shared", "first", 20 * time.Second, true}, + {"three", "shared", "second", 2 * time.Second, false}, + } { + events = append(events, map[string]any{"type": "test", "content": map[string]any{ + "span_id": i + 1, "duration": int64(tc.duration), "meta": map[string]any{ + "test.module": tc.module, "test.suite": tc.suite, "test.name": tc.name, "test.status": "pass", "test.is_retry": tc.retry, + }, + }}) + } + payload, err := msgp.AppendIntf(nil, map[string]any{"events": events}) + require.NoError(t, err) + facts, err := serverWithCoverage(t, payload).Facts() + require.NoError(t, err) + require.Equal(t, time.Second, facts.SuiteDurationMedian) + require.Equal(t, []SlowSuite{{Module: "three", Suite: "shared", Duration: 6 * time.Second}}, facts.SlowSuites) +} diff --git a/internal/testdrive/intake/server.go b/internal/testdrive/intake/server.go index 082a6a54..6acf0ebb 100644 --- a/internal/testdrive/intake/server.go +++ b/internal/testdrive/intake/server.go @@ -48,7 +48,6 @@ type storedRequest struct { Timestamp time.Time `json:"timestamp"` ContentEncoding string `json:"content_encoding,omitempty"` DecodeError string `json:"decode_error,omitempty"` - RawBody []byte `json:"raw_body,omitempty"` Method string `json:"method"` Path string `json:"path"` ContentType string `json:"content_type,omitempty"` @@ -170,9 +169,6 @@ func (s *Server) recordRequests(next http.Handler) http.Handler { } if decodeErr != nil { stored.DecodeError = decodeErr.Error() - stored.RawBody = rawRequest.Body - } else if !utf8.Valid(body) { - stored.RawBody = rawRequest.Body } s.requestsMu.Lock() persistErr := s.persistRequest(len(s.requests)+1, stored) diff --git a/internal/testdrive/intake/server_test.go b/internal/testdrive/intake/server_test.go index b804ac42..a17c00dc 100644 --- a/internal/testdrive/intake/server_test.go +++ b/internal/testdrive/intake/server_test.go @@ -152,6 +152,7 @@ func TestServerStoresMessagePackAsJSON(t *testing.T) { require.NoError(t, err) require.True(t, json.Valid(storedBytes)) require.Contains(t, string(storedBytes), `"events": []`) + require.NotContains(t, string(storedBytes), `"raw_body"`) } func TestServerStoresAndRecognizesGzippedMessagePack(t *testing.T) { @@ -240,6 +241,19 @@ func TestDecodeMultipartStoresEveryPartAsJSON(t *testing.T) { require.Equal(t, "events.msgpack", stored.Parts[1].Filename) require.JSONEq(t, `{"count":2}`, string(stored.Parts[1].Body)) require.JSONEq(t, `"hello"`, string(stored.Parts[2].Body)) + + server := &Server{directory: t.TempDir()} + request := httptest.NewRequest(http.MethodPost, "/observed", bytes.NewReader(body.Bytes())) + request.Header.Set("Content-Type", writer.FormDataContentType()) + response := httptest.NewRecorder() + server.recordRequests(newHandler()).ServeHTTP(response, request) + require.Equal(t, http.StatusOK, response.Code) + data, err := os.ReadFile(filepath.Join(server.directory, "001-request.json")) + require.NoError(t, err) + require.NotContains(t, string(data), `"raw_body"`) + var persisted storedRequest + require.NoError(t, json.Unmarshal(data, &persisted)) + require.JSONEq(t, string(decoded), string(persisted.Body)) } func TestRequestDecodingRejectsMalformedPayloads(t *testing.T) { @@ -334,7 +348,7 @@ func TestGzippedSettingsRequest(t *testing.T) { require.JSONEq(t, string(payload), string(stored.Body)) } -func TestFailedAndBinaryRequestsRemainOnDisk(t *testing.T) { +func TestStoredRequestsOmitRawBody(t *testing.T) { for _, tc := range []struct { name, contentType, encoding string body []byte @@ -365,13 +379,12 @@ func TestFailedAndBinaryRequestsRemainOnDisk(t *testing.T) { require.NoError(t, err) var stored struct { Path string `json:"path"` - RawBody []byte `json:"raw_body"` DecodeError string `json:"decode_error"` ContentEncoding string `json:"content_encoding"` } require.NoError(t, json.Unmarshal(data, &stored)) require.Equal(t, "/observed", stored.Path) - require.Equal(t, tc.body, stored.RawBody) + require.NotContains(t, string(data), `"raw_body"`) require.Equal(t, tc.encoding, stored.ContentEncoding) if tc.decodeError != "" { require.Contains(t, stored.DecodeError, tc.decodeError) diff --git a/internal/testdrive/report.go b/internal/testdrive/report.go index 1887cdde..48a418e8 100644 --- a/internal/testdrive/report.go +++ b/internal/testdrive/report.go @@ -1,30 +1,150 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + package testdrive import ( + _ "embed" "errors" "fmt" - "github.com/DataDog/ddtest/internal/testdrive/intake" "html/template" "net/url" "os" "path/filepath" + "slices" + "sort" "strings" "time" + + "github.com/DataDog/ddtest/internal/testdrive/intake" ) const reportFilename = "report.html" -type reportRuntime struct{ Framework, Tracer string } -type reportFact struct{ Label, Value string } type reportCard struct { - Title, Context string - Tests []intake.Test - Coverages []intake.CoverageFact + Kind string + Title string + Count int + Context string + Tests []reportTest + Suites []reportSuite + Coverages []reportCoverage } + +type reportTest struct { + Label string + Name string + Suite string + SourceFile string + Status string + Duration string + Attempts []reportAttempt + CoverageLevel string + CoveredFiles []string + Source reportSource + DurationNanos int64 +} + +type reportAttempt struct { + Number int + Status string + Duration string + Kind string + ErrorType string + ErrorMessage string + ErrorStack string +} + +type reportCoverage struct { + Name string + Level string + FileCount int + Files []string + SourceFile string + Source reportSource +} + +type reportSource struct { + Start int + End int + Lines []reportSourceLine + Error string +} + +type reportSourceLine struct { + Number int + Code template.HTML +} + +type reportSuite struct { + Key string + Name string + Status string + Duration string + TestCount int + CoveredCount int + ShowCoverage bool + CoveredFiles []string + Tests []reportSuiteTest + DurationNanos int64 +} + +type reportSuiteTest struct { + TestIndex int + Name string + Status string + Duration string +} + +type reportFact struct { + Label string + Value string + Tone string +} + +type reportArtifact struct { + Title string + Href string +} + type reportModel struct { - Headline, Summary string - Facts []reportFact - Cards []reportCard + NoTestEvents bool + CoverageLevel string + Runtime reportRuntime + Summary string + Cards []reportCard + Facts []reportFact + Artifacts []reportArtifact + Suites []reportSuite + Tests []reportTest +} + +type reportRuntime struct{ Framework, Tracer, Command, Output, Error string } + +type reportSourceFile struct { + lines []string + err error +} + +type reportSourceKey struct { + file string + start, end int +} + +type reportBuilder struct { + repositoryRoot string + sourceFiles map[string]reportSourceFile + sources map[reportSourceKey]reportSource +} + +func newReportBuilder(repositoryRoot string) *reportBuilder { + return &reportBuilder{ + repositoryRoot: repositoryRoot, + sourceFiles: make(map[string]reportSourceFile), + sources: make(map[reportSourceKey]reportSource), + } } func writeReport(repositoryRoot, sessionDirectory string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) (string, error) { @@ -33,56 +153,115 @@ func writeReport(repositoryRoot, sessionDirectory string, findings intake.Facts, if err != nil { return "", fmt.Errorf("create testdrive report: %w", err) } - executeErr := testdriveReport.Execute(file, buildReport(repositoryRoot, findings, commandFailed, runtime...)) - if err := errors.Join(executeErr, file.Close()); err != nil { + + model := buildReport(repositoryRoot, findings, commandFailed, runtime...) + executeErr := testdriveReport.Execute(file, model) + closeErr := file.Close() + if err := errors.Join(executeErr, closeErr); err != nil { return "", fmt.Errorf("write testdrive report: %w", err) } return path, nil } -func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) reportModel { - info := reportRuntime{Framework: "Test command", Tracer: "Isolated installation"} + +func buildReport(repositoryRoot string, findings intake.Facts, commandFailed bool, runtime ...reportRuntime) reportModel { + builder := newReportBuilder(repositoryRoot) + info := reportRuntime{Framework: "Test command", Tracer: "Not reported"} if len(runtime) > 0 { info = runtime[0] } - coverage := "Not reported" - if findings.CoveredTestCount > 0 { - coverage = fmt.Sprintf("%d / %d", findings.CoveredTestCount, findings.TestCount) - } status := passedFailed(!commandFailed) if findings.TestEventCount == 0 { status = "No test results received" } - model := reportModel{Headline: "Test events received.", Summary: "No findings.", Facts: []reportFact{ - {"Test events", fmt.Sprint(findings.TestEventCount)}, - {"Tests with coverage", coverage}, - {info.Framework, status}, - {"Datadog library", info.Tracer}, - }} + showTestCoverage := findings.CoverageLevel == "test" + showSuiteCoverage := findings.CoverageLevel == "suite" + suites := builder.reportSuites(findings.Tests, findings.SuiteCoverages, showSuiteCoverage) + coverageLabel := "Tests with coverage" + covered, total := findings.CoveredTestCount, findings.TestCount + if showSuiteCoverage { + coverageLabel = "Suites with coverage" + covered, total = 0, len(suites) + for _, suite := range suites { + if suite.CoveredCount > 0 { + covered++ + } + } + } + coverageValue := fmt.Sprintf("%d / %d", covered, total) + if covered == 0 { + coverageValue = "Not reported" + } + model := reportModel{ + Summary: "No findings.", + NoTestEvents: findings.TestEventCount == 0, + Runtime: info, + CoverageLevel: findings.CoverageLevel, + Facts: []reportFact{ + {Label: info.Framework, Value: status, Tone: factTone(!commandFailed && findings.TestEventCount > 0)}, + {Label: "Datadog library", Value: info.Tracer, Tone: "good"}, + {Label: coverageLabel, Value: coverageValue, Tone: factTone(total > 0 && covered == total)}, + }, + Artifacts: []reportArtifact{ + {Title: "JSON traffic", Href: "intake/"}, + {Title: "Test output", Href: testOutputFilename}, + }, + Tests: builder.reportTests(findings.Tests, showTestCoverage), + } + model.Suites = suites + if findings.TestEventCount == 0 { - model.Headline = "No test events received." model.Summary = "Check the instrumentation setup." } if findings.EmptyCoverageEntryCount > 0 { model.Cards = append(model.Cards, reportCard{ + Kind: "error", Title: "Tracer error: empty coverage entries", + Count: findings.EmptyCoverageEntryCount, Context: fmt.Sprintf("%d coverage entries had an empty files list. Affected payloads were excluded from coverage counts. Inspect the captured traffic.", findings.EmptyCoverageEntryCount), }) } if len(findings.FailedTests) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any tests failed?", Tests: findings.FailedTests}) + model.Cards = append(model.Cards, reportCard{ + Kind: "failed", Title: "Failed tests", Context: "Inspect the errors and source behind each failure.", Count: len(findings.FailedTests), + Tests: builder.reportTests(findings.FailedTests, showTestCoverage), + }) } if len(findings.FlakyTests) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any flaky tests?", Tests: findings.FlakyTests}) + model.Cards = append(model.Cards, reportCard{ + Kind: "flaky", Title: "Flaky tests", Context: "These tests both passed and failed across attempts.", Count: len(findings.FlakyTests), + Tests: builder.reportTests(findings.FlakyTests, showTestCoverage), + }) } if len(findings.SlowTests) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any tests slower than the others?", Context: "Median test time · " + formatDuration(findings.TestDurationMedian), Tests: findings.SlowTests}) + model.Cards = append(model.Cards, reportCard{ + Kind: "slow", Title: "Slow tests", Count: len(findings.SlowTests), + Context: "Median test time · " + formatDuration(findings.TestDurationMedian), + Tests: builder.reportTests(findings.SlowTests, showTestCoverage), + }) + } + if len(findings.SlowSuites) > 0 { + byKey := make(map[string]reportSuite, len(model.Suites)) + for _, suite := range model.Suites { + byKey[suite.Key] = suite + } + card := reportCard{Kind: "slow", Title: "Slow suites", Count: len(findings.SlowSuites), Context: "Median suite time · " + formatDuration(findings.SuiteDurationMedian)} + for _, suite := range findings.SlowSuites { + if row, ok := byKey[suite.Module+"\x00"+suite.Suite]; ok { + card.Suites = append(card.Suites, row) + } + } + model.Cards = append(model.Cards, card) } if len(findings.BroadCoverage) > 0 { - model.Cards = append(model.Cards, reportCard{Title: "Any unusually broad test coverage?", Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), Coverages: findings.BroadCoverage}) + model.Cards = append(model.Cards, reportCard{ + Kind: "coverage", Title: "Broad coverage", Count: len(findings.BroadCoverage), + Context: fmt.Sprintf("Median covered files · %d", findings.CoveredFilesMedian), + Coverages: builder.reportCoverages(findings.BroadCoverage), + }) } count := len(findings.ConfigurationErrors) + findings.EmptyCoverageEntryCount for _, size := range []int{ - len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.BroadCoverage), + len(findings.FailedTests), len(findings.FlakyTests), len(findings.SlowTests), len(findings.SlowSuites), len(findings.BroadCoverage), } { count += size } @@ -95,15 +274,153 @@ func buildReport(_ string, findings intake.Facts, commandFailed bool, runtime .. return model } -var testdriveReport = template.Must(template.New("testdrive-report").Parse(` -DDTest report - -

{{.Headline}}

{{.Summary}}

-{{range .Cards}}

{{.Title}}

{{.Context}}

{{end}} -

Run details

{{range .Facts}}
{{.Label}}
{{.Value}}
{{end}}
-

Artifacts

`)) +func (builder *reportBuilder) reportTests(findings []intake.Test, showCoverage bool) []reportTest { + tests := make([]reportTest, 0, len(findings)) + for _, finding := range findings { + name := finding.Name + if finding.Parameters != "" { + name += " " + finding.Parameters + } + label := name + if finding.Suite != "" { + label = finding.Suite + " › " + label + } + if finding.Module != "" { + label = finding.Module + " › " + label + } + status := testDisplayStatus(finding) + suite := finding.Suite + if finding.Module != "" { + suite = finding.Module + " › " + suite + } + test := reportTest{ + Label: label, Name: name, Suite: suite, + SourceFile: finding.SourceFile, Status: status, + Duration: formatDuration(findingDuration(finding)), + DurationNanos: int64(findingDuration(finding)), + Attempts: make([]reportAttempt, 0, len(finding.Attempts)), + Source: builder.readSource(finding.SourceFile, finding.SourceStart, finding.SourceEnd), + } + if showCoverage && finding.CoverageLevel == "test" { + test.CoverageLevel = finding.CoverageLevel + test.CoveredFiles = finding.CoveredFiles + } + for attemptIndex, attempt := range finding.Attempts { + kind := "Initial run" + if attempt.Retry { + kind = "Retry" + if attempt.RetryReason != "" { + kind += " · " + strings.ReplaceAll(attempt.RetryReason, "_", " ") + } + } + test.Attempts = append(test.Attempts, reportAttempt{ + Number: attemptIndex + 1, Status: displayStatus(attempt.Status), + Duration: formatDuration(attempt.Duration), Kind: kind, + ErrorType: attempt.ErrorType, ErrorMessage: attempt.ErrorMessage, ErrorStack: attempt.ErrorStack, + }) + } + tests = append(tests, test) + } + return tests +} + +func (builder *reportBuilder) reportCoverages(findings []intake.CoverageFact) []reportCoverage { + coverages := make([]reportCoverage, 0, len(findings)) + for _, finding := range findings { + files := slices.Clone(finding.Files) + slices.Sort(files) + coverage := reportCoverage{ + Name: finding.Name, Level: finding.Level, FileCount: finding.FileCount, + Files: files, SourceFile: finding.SourceFile, + } + if finding.Level == "test" { + coverage.Source = builder.readSource(finding.SourceFile, finding.SourceStart, finding.SourceEnd) + } + coverages = append(coverages, coverage) + } + return coverages +} + +func (builder *reportBuilder) reportSuites(tests []intake.Test, coverages []intake.SuiteCoverage, showCoverage bool) []reportSuite { + byName := make(map[string]*reportSuite) + durations := make(map[string]time.Duration) + for index, test := range tests { + key := test.Module + "\x00" + test.Suite + name := test.Suite + if name == "" { + name = "Unknown suite" + } + suite, found := byName[key] + status := testDisplayStatus(test) + if !found { + if test.Module != "" { + name = test.Module + " › " + name + } + suite = &reportSuite{Key: key, Name: name, Status: suiteStatus(status)} + byName[key] = suite + } else if suiteStatusRank(status) > suiteStatusRank(suite.Status) { + suite.Status = suiteStatus(status) + } + suite.TestCount++ + durations[key] += findingDuration(test) + testName := test.Name + if test.Parameters != "" { + testName += " " + test.Parameters + } + suite.Tests = append(suite.Tests, reportSuiteTest{ + TestIndex: index, Name: testName, Status: status, + Duration: formatDuration(findingDuration(test)), + }) + } + if showCoverage { + for _, coverage := range coverages { + key := coverage.Module + "\x00" + coverage.Suite + if suite, found := byName[key]; found { + files := slices.Clone(coverage.Files) + slices.Sort(files) + suite.ShowCoverage = true + suite.CoveredCount = coverage.CoveredTests + suite.CoveredFiles = files + } + } + } + + suites := make([]reportSuite, 0, len(byName)) + for key, suite := range byName { + suite.Duration = formatDuration(durations[key]) + suite.DurationNanos = int64(durations[key]) + suites = append(suites, *suite) + } + sort.Slice(suites, func(i, j int) bool { return suites[i].Name < suites[j].Name }) + return suites +} + +func suiteStatus(status string) string { + switch status { + case "Fail": + return "Failed" + case "Pass": + return "Passed" + } + return status +} + +func suiteStatusRank(status string) int { + switch suiteStatus(status) { + case "Failed": + return 4 + case "Flaky": + return 3 + case "Passed", "Pass": + return 2 + case "Skip": + return 1 + default: + return 0 + } +} -func testDisplayStatus(test intake.Test) (string, string) { +func testDisplayStatus(test intake.Test) string { status := test.Status sawPass := status == "pass" sawFailure := status == "fail" @@ -112,19 +429,12 @@ func testDisplayStatus(test intake.Test) (string, string) { sawFailure = sawFailure || attempt.Status == "fail" } if sawPass && sawFailure { - return "Flaky", "attention" + return "Flaky" } if status == "" && len(test.Attempts) > 0 { status = test.Attempts[len(test.Attempts)-1].Status } - return displayStatus(status), attemptTone(status) -} - -func attemptTone(status string) string { - if status == "pass" { - return "good" - } - return "attention" + return displayStatus(status) } func findingDuration(test intake.Test) time.Duration { @@ -134,6 +444,187 @@ func findingDuration(test intake.Test) time.Duration { return test.Attempts[0].Duration } +func readSource(repositoryRoot, sourceFile string, sourceStart, sourceEnd int) reportSource { + return newReportBuilder(repositoryRoot).readSource(sourceFile, sourceStart, sourceEnd) +} + +func (builder *reportBuilder) readSource(sourceFile string, sourceStart, sourceEnd int) reportSource { + if sourceFile == "" { + return reportSource{Error: "Source file not reported."} + } + if sourceStart < 1 { + return reportSource{Error: "Source line not reported."} + } + key := reportSourceKey{file: sourceFile, start: sourceStart, end: sourceEnd} + if source, cached := builder.sources[key]; cached { + return source + } + path := sourceFile + if !filepath.IsAbs(path) { + path = filepath.Join(builder.repositoryRoot, filepath.FromSlash(path)) + } + file, cached := builder.sourceFiles[path] + if !cached { + contents, err := os.ReadFile(path) + file.err = err + if err == nil && len(contents) > 0 { + text := strings.TrimSuffix(strings.ReplaceAll(string(contents), "\r\n", "\n"), "\n") + file.lines = strings.Split(text, "\n") + } + builder.sourceFiles[path] = file + } + if file.err != nil { + source := reportSource{Error: "Source could not be read: " + file.err.Error()} + builder.sources[key] = source + return source + } + lines := file.lines + if sourceStart > len(lines) { + source := reportSource{Error: fmt.Sprintf("Source line %d is outside %s.", sourceStart, sourceFile)} + builder.sources[key] = source + return source + } + if sourceEnd < sourceStart { + sourceEnd = sourceStart + 4 + } + sourceEnd = min(sourceEnd, len(lines)) + source := reportSource{Start: sourceStart, End: sourceEnd} + inBlockComment := false + for lineIndex := sourceStart - 1; lineIndex < sourceEnd; lineIndex++ { + code := highlightJavaScriptLine(lines[lineIndex], &inBlockComment) + if filepath.Ext(path) == ".py" || filepath.Ext(path) == ".rb" { + code = template.HTML(template.HTMLEscapeString(lines[lineIndex])) + } + source.Lines = append(source.Lines, reportSourceLine{ + Number: lineIndex + 1, + Code: code, + }) + } + builder.sources[key] = source + return source +} + +func highlightJavaScriptLine(line string, inBlockComment *bool) template.HTML { + var highlighted strings.Builder + for index := 0; index < len(line); { + if *inBlockComment { + end := strings.Index(line[index:], "*/") + if end < 0 { + writeToken(&highlighted, "comment", line[index:]) + break + } + end += index + 2 + writeToken(&highlighted, "comment", line[index:end]) + *inBlockComment = false + index = end + continue + } + if strings.HasPrefix(line[index:], "//") { + writeToken(&highlighted, "comment", line[index:]) + break + } + if strings.HasPrefix(line[index:], "/*") { + end := strings.Index(line[index+2:], "*/") + if end < 0 { + writeToken(&highlighted, "comment", line[index:]) + *inBlockComment = true + break + } + end += index + 4 + writeToken(&highlighted, "comment", line[index:end]) + index = end + continue + } + if line[index] == '\'' || line[index] == '"' || line[index] == '`' { + end := stringEnd(line, index) + writeToken(&highlighted, "string", line[index:end]) + index = end + continue + } + if isIdentifierStart(line[index]) { + end := index + 1 + for end < len(line) && isIdentifierPart(line[end]) { + end++ + } + word := line[index:end] + class := "" + if javascriptKeywords[word] { + class = "keyword" + } else if javascriptLiterals[word] { + class = "literal" + } + writeToken(&highlighted, class, word) + index = end + continue + } + if line[index] >= '0' && line[index] <= '9' { + end := index + 1 + for end < len(line) && ((line[end] >= '0' && line[end] <= '9') || line[end] == '.') { + end++ + } + writeToken(&highlighted, "number", line[index:end]) + index = end + continue + } + writeToken(&highlighted, "", line[index:index+1]) + index++ + } + return template.HTML(highlighted.String()) +} + +func stringEnd(line string, start int) int { + quote := line[start] + escaped := false + for index := start + 1; index < len(line); index++ { + if escaped { + escaped = false + continue + } + if line[index] == '\\' { + escaped = true + continue + } + if line[index] == quote { + return index + 1 + } + } + return len(line) +} + +func writeToken(output *strings.Builder, class, value string) { + escaped := template.HTMLEscapeString(value) + if class == "" { + output.WriteString(escaped) + return + } + output.WriteString(``) + output.WriteString(escaped) + output.WriteString(``) +} + +func isIdentifierStart(character byte) bool { + return character == '_' || character == '$' || character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' +} + +func isIdentifierPart(character byte) bool { + return isIdentifierStart(character) || character >= '0' && character <= '9' +} + +var javascriptKeywords = map[string]bool{ + "async": true, "await": true, "break": true, "case": true, "catch": true, "class": true, + "const": true, "continue": true, "default": true, "delete": true, "do": true, "else": true, + "export": true, "extends": true, "finally": true, "for": true, "from": true, "function": true, + "if": true, "import": true, "in": true, "instanceof": true, "let": true, "new": true, + "of": true, "return": true, "switch": true, "throw": true, "try": true, "typeof": true, + "var": true, "void": true, "while": true, "with": true, "yield": true, +} + +var javascriptLiterals = map[string]bool{ + "false": true, "null": true, "true": true, "undefined": true, +} + func passedFailed(passed bool) string { if passed { return "Passed" @@ -141,6 +632,29 @@ func passedFailed(passed bool) string { return "Failed" } +func factTone(good bool) string { + if good { + return "good" + } + return "attention" +} + +// reportStatus provides consistent labels for the report's test and suite filters. +func reportStatus(status string) string { + switch status { + case "Pass", "Passed": + return "Passed" + case "Fail", "Failed": + return "Failed" + case "Skip", "Skipped": + return "Skipped" + case "Flaky": + return "Flaky" + default: + return "Unknown" + } +} + func displayStatus(status string) string { if status == "" { return "Unknown" @@ -155,13 +669,6 @@ func formatDuration(duration time.Duration) string { return duration.Round(time.Millisecond).String() } -func plural(count int, singular, plural string) string { - if count == 1 { - return singular - } - return plural -} - func fileURL(path string) (string, error) { absolutePath, err := filepath.Abs(path) if err != nil { @@ -190,3 +697,15 @@ func absoluteFileURL(absolutePath string) string { func terminalLink(target, label string) string { return "\x1b]8;;" + target + "\x1b\\" + label + "\x1b]8;;\x1b\\" } + +func plural(count int, singular, plural string) string { + if count == 1 { + return singular + } + return plural +} + +//go:embed report.html +var reportHTML string + +var testdriveReport = template.Must(template.New("testdrive-report").Funcs(template.FuncMap{"plural": plural, "reportStatus": reportStatus}).Parse(reportHTML)) diff --git a/internal/testdrive/report.html b/internal/testdrive/report.html new file mode 100644 index 00000000..3cb8a706 --- /dev/null +++ b/internal/testdrive/report.html @@ -0,0 +1,1901 @@ +{{ define "suite-detail" }} +
+ {{ if .ShowCoverage }} +
+ + Suite coverage · {{ len .CoveredFiles }} + {{ plural (len .CoveredFiles) "file" "files" }} + + {{ template "covered-files" .CoveredFiles }} +
+ {{ end }} +
+ Tests · {{ len .Tests }} {{ plural (len .Tests) "test" "tests" }} + + + + + + + + + + {{ range .Tests }} + + + + + + {{ end }} + +
TestStatusDuration
+ + {{ template "status" .Status }}{{ .Duration }}
+
+
+{{ end }} + +{{ define "icon-arrow-repeat" }} + +{{ end }} +{{ define "icon-box-arrow-up-right" }} + +{{ end }} +{{ define "icon-check-circle-fill" }} + +{{ end }} +{{ define "icon-chevron-down" }} + +{{ end }} +{{ define "icon-chevron-right" }} + +{{ end }} +{{ define "icon-exclamation-circle" }} + +{{ end }} +{{ define "icon-file-earmark-text" }} + +{{ end }} +{{ define "icon-question-circle" }} + +{{ end }} +{{ define "icon-search" }} + +{{ end }} +{{ define "icon-skip-forward-circle" }} + +{{ end }} +{{ define "icon-stopwatch" }} + +{{ end }} +{{ define "icon-x-circle-fill" }} + +{{ end }} +{{ define "status" }} + {{ if eq (reportStatus .) "Passed" }} + {{ template "icon-check-circle-fill" }} + {{ else if eq (reportStatus .) "Failed" }} + {{ template "icon-x-circle-fill" }} + {{ else if eq (reportStatus .) "Flaky" }} + {{ template "icon-arrow-repeat" }} + {{ else if eq (reportStatus .) "Skipped" }} + {{ template "icon-skip-forward-circle" }} + {{ else }} + {{ template "icon-question-circle" }} + {{ end }}{{ reportStatus . }} +{{ end }} +{{ define "attempts" }} +
+

Attempts

+ {{ range .Attempts }} +
+
+ {{ template "status" .Status }}Run {{ .Number }} · {{ .Kind }}{{ .Duration }} +
+ {{ if .ErrorMessage }} + + 
{{if .ErrorType}}{{.ErrorType}}: {{end}}{{.ErrorMessage}}
+ {{ end }} + {{ if .ErrorStack }} +
+ Stack trace + 
{{ .ErrorStack }}
+
+ {{ end }} +
+ {{ else }} +

No attempts reported.

+ {{ end }} +
+{{ end }} +{{ define "source" }} +
+

+ {{ if .Lines }} + Source · lines {{ .Start }}–{{ .End }} + {{ else }} + Source + {{ end }} +

+ {{ if .Lines }} + + 
{{range .Lines}}{{.Number}}{{.Code}}{{end}}
+ {{ else if .Error }} +

{{ .Error }}

+ {{ end }} +
+{{ end }} +{{ define "covered-files" }} + {{ if . }} +
+ +
+ +
+
+ {{ end }} +{{ end }} +{{ define "test-detail" }} +
+
+ {{ template "attempts" . }} +
+ {{ if .SourceFile }} +

{{ .SourceFile }}

+ {{ end }}{{ template "source" .Source }} +
+
+ {{ if .CoverageLevel }} +
+ + Test coverage · {{ len .CoveredFiles }} + {{ plural (len .CoveredFiles) "file" "files" }} + + {{ template "covered-files" .CoveredFiles }} +
+ {{ end }} +
+{{ end }} +{{ define "pager" }} +
+ +
+ +
+
+{{ end }} + + + + + + ddtest · Test report + + + +
+
+
+ ddtest +

Test report

+
+

+ {{ .Runtime.Framework }} · {{ .Summary }} +

+
+
+
+ {{ range .Artifacts }} + {{ template "icon-box-arrow-up-right" }}{{ .Title }} + {{ end }} +
+
+
+
+ {{ if .Cards }} +
+ {{ range $i, $card := .Cards }} +
+ +
+ {{ end }} +
+ {{ range $i, $card := .Cards }} + + {{ end }} + {{ else }} +
+ {{ if eq .Summary "No findings." }} + {{ template "icon-check-circle-fill" }}No findings in this run. + Explore the results below. + {{ else }} + {{ .Summary }} + {{ end }} +
+ {{ end }} +
+

Run details

+

{{ len .Tests }} {{ plural (len .Tests) "test" "tests" }} · {{ len .Suites }} {{ plural (len .Suites) "suite" "suites" }}

+ {{ if .NoTestEvents }}

No test events received.

{{ end }} +
+ {{ range .Facts }} +
+
{{ .Label }}
+
{{ .Value }}
+
+ {{ end }} +
+
+ {{ .Runtime.Framework }} output · expand logs + {{ if .Runtime.Command }}

{{ .Runtime.Command }}

{{ end }} + {{ if .Runtime.Error }}

Command failed: {{ .Runtime.Error }}

{{ end }} + {{ if .Runtime.Output }} + 
{{ .Runtime.Output }}
+ {{ else }} +

The command produced no output.

+ {{ end }} + Open full output file +
+
+ +
+
+ +
+ +
+ + +
+
+ + + + + + + + + + + {{ if eq .CoverageLevel "test" }} + + {{ end }} + + + + {{ range $i, $test := .Tests }} + + + + + + + + + {{ if eq $.CoverageLevel "test" }} + + {{ end }} + + + + + + + {{ end }} +
+ + + + + + + + + + + + + + + +
{{ $i }}{{ template "status" .Status }} + + +
{{ .SourceFile }}
+
{{ .Suite }}
+ 		{{ .Duration }}{{ len .Attempts }} + {{ if .CoverageLevel }} + {{ len .CoveredFiles }} files + {{ else }} + Not reported + {{ end }} + + {{ template "icon-chevron-down" }} +
+
+ + {{ template "pager" }} +
+
+ + + diff --git a/internal/testdrive/report_test.go b/internal/testdrive/report_test.go index b7e71ab0..b9d88e0f 100644 --- a/internal/testdrive/report_test.go +++ b/internal/testdrive/report_test.go @@ -1,10 +1,18 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + package testdrive import ( "bytes" + "fmt" "os" + "path/filepath" "strings" "testing" + "time" "github.com/DataDog/ddtest/internal/testdrive/intake" ) @@ -42,33 +50,355 @@ func TestReportCountsIndividualFindings(t *testing.T) { } } -func TestStaticReportEscapesFindingsAndDescribesMissingCoverage(t *testing.T) { - path, err := writeReport(t.TempDir(), t.TempDir(), intake.Facts{TestEventCount: 1, TestCount: 1, FailedTests: []intake.Test{{Name: ""}}}, true) - if err != nil { +func TestReportKeepsPythonAndRubySourceEscaped(t *testing.T) { + for _, extension := range []string{".py", ".rb"} { + t.Run(extension, func(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "test"+extension), []byte("value = ''\nassert value\n"), 0644); err != nil { + t.Fatal(err) + } + source := readSource(root, "test"+extension, 1, 0) + if source.Error != "" || len(source.Lines) < 2 { + t.Fatalf("source not rendered: %+v", source) + } + if !strings.Contains(string(source.Lines[0].Code), "<script>") || strings.Contains(string(source.Lines[0].Code), "`, Module: "one", Suite: "shared", Parameters: `{"case":1}`, Status: "pass"}, + {Name: `same `, Module: "two", Suite: "shared", Parameters: `{"case":2}`, Status: "fail"}, + } + model := buildReport(t.TempDir(), intake.Facts{Tests: tests}, false) + if model.Tests[0].Name == model.Tests[1].Name || model.Tests[0].Suite == model.Tests[1].Suite { + t.Fatal("table loses module or parameter identity") + } + report := renderTestReport(t, t.TempDir(), intake.Facts{Tests: tests}) + for _, want := range []string{`data-status="Passed"`, `data-status="Failed"`, `aria-controls="test-detail-0"`, `aria-controls="test-detail-1"`, `id="test-detail-0"`, `id="test-detail-1"`, `<script>alert(1)</script>`} { + if !strings.Contains(report, want) { + t.Errorf("report missing %q", want) + } + } + if strings.Contains(report, `same \nTest Suites: 1 failed, 1 total\n", events: 1, want: "Test Suites: 1 failed, 1 total"}, {name: "80 lines", output: strings.Repeat("log line\n", 79) + "final failure", events: 1, want: "final failure"}, {name: "long output", output: "initial failure\n" + strings.Repeat("log line\n", 79) + "final failure\n", events: 1, want: "... 1 line omitted; see the full test output below ..."}, } { @@ -395,6 +400,24 @@ func TestRunStillReportsEventsWhenJestFails(t *testing.T) { if string(contents) != tc.output { t.Fatalf("saved output changed: %q", contents) } + report, err := os.ReadFile(filepath.Join(installer.sessionDirectory, reportFilename)) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`
`, "Jest output", "npx jest", "Command failed: exit status 1", tc.output} { + if !strings.Contains(html.UnescapeString(string(report)), want) { + t.Errorf("report missing full command output or failure context %q", want) + } + } + if tc.output == "" && !strings.Contains(string(report), "The command produced no output.") { + t.Fatal("report hides empty command output") + } + if strings.Contains(string(report), "") { + t.Fatal("command output can execute HTML") + } + if strings.Contains(string(report), "
Test events
") { + t.Fatal("run details still shows the event count") + } if strings.Contains(output.String(), "Full test output: "+repositoryRoot) { t.Fatal("output path is absolute") }