From a2de4573d3fbf33e700e2cb5c1ab80092c0646eb Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 29 Sep 2026 14:37:12 +0530 Subject: [PATCH 1/2] fix: suite_summary.py exits 1 when any suite has failing tests Refs #12 --- scripts/suite_summary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/suite_summary.py b/scripts/suite_summary.py index 4b5d13a..5047a0f 100644 --- a/scripts/suite_summary.py +++ b/scripts/suite_summary.py @@ -19,7 +19,8 @@ Each argument is `Label=path`. Missing or unparseable files are reported as such rather than skipped silently — a suite whose report never arrived is a -result worth seeing, not a blank row. Shard globs are supported so the E2E +result worth seeing, not a blank row. Exits 1 when any suite has failing +tests, so CI can gate on it. Shard globs are supported so the E2E matrix can be passed as one label: python scripts/suite_summary.py "E2E=reports/e2e_shard_*.json" @@ -112,9 +113,8 @@ def main(argv: list[str]) -> int: print("\n".join(rows)) print() if any_failure: - print("> One or more suites have failing tests. Job status alone does not") - print("> reflect this — every suite runs with `continue-on-error: true`.") - return 0 + print("> One or more suites have failing tests.") + return 1 if any_failure else 0 if __name__ == "__main__": From 56fc0926faad4c91bc51b9b73dcec96a10ada093 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 29 Sep 2026 14:37:12 +0530 Subject: [PATCH 2/2] fix(ci): fail the run when any suite has failing tests Every pytest step is continue-on-error, so shards went green with 1-7 failed tests. Test Summary now fails the run from the real JSON counts. Refs #12 --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7738b34..b62e4e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -712,7 +712,8 @@ jobs: "Accessibility=suite-artifacts/accessibility.json" \ "Visual Regression=suite-artifacts/reports/visual.json" \ "Performance=suite-artifacts/performance.json" \ - "Load=suite-artifacts/load.json" + "Load=suite-artifacts/load.json" \ + || echo "TESTS_FAILED=1" >> "$GITHUB_ENV" echo "" echo "**Platform:** ${{ env.BASE_URL }}" echo "" @@ -739,3 +740,11 @@ jobs: reports/e2e_combined.json reports/e2e_summary.md retention-days: 30 + + # Every suite's pytest step is `continue-on-error` so all reports get + # produced; this is where a failing test finally turns the run red. + - name: Fail on test failures + if: env.TESTS_FAILED == '1' + run: | + echo "::error::One or more suites have failing tests — see the results table in the job summary." + exit 1