diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7738b34..b62e4e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -712,7 +712,8 @@ jobs: "Accessibility=suite-artifacts/accessibility.json" \ "Visual Regression=suite-artifacts/reports/visual.json" \ "Performance=suite-artifacts/performance.json" \ - "Load=suite-artifacts/load.json" + "Load=suite-artifacts/load.json" \ + || echo "TESTS_FAILED=1" >> "$GITHUB_ENV" echo "" echo "**Platform:** ${{ env.BASE_URL }}" echo "" @@ -739,3 +740,11 @@ jobs: reports/e2e_combined.json reports/e2e_summary.md retention-days: 30 + + # Every suite's pytest step is `continue-on-error` so all reports get + # produced; this is where a failing test finally turns the run red. + - name: Fail on test failures + if: env.TESTS_FAILED == '1' + run: | + echo "::error::One or more suites have failing tests — see the results table in the job summary." + exit 1 diff --git a/scripts/suite_summary.py b/scripts/suite_summary.py index 4b5d13a..5047a0f 100644 --- a/scripts/suite_summary.py +++ b/scripts/suite_summary.py @@ -19,7 +19,8 @@ Each argument is `Label=path`. Missing or unparseable files are reported as such rather than skipped silently — a suite whose report never arrived is a -result worth seeing, not a blank row. Shard globs are supported so the E2E +result worth seeing, not a blank row. Exits 1 when any suite has failing +tests, so CI can gate on it. Shard globs are supported so the E2E matrix can be passed as one label: python scripts/suite_summary.py "E2E=reports/e2e_shard_*.json" @@ -112,9 +113,8 @@ def main(argv: list[str]) -> int: print("\n".join(rows)) print() if any_failure: - print("> One or more suites have failing tests. Job status alone does not") - print("> reflect this — every suite runs with `continue-on-error: true`.") - return 0 + print("> One or more suites have failing tests.") + return 1 if any_failure else 0 if __name__ == "__main__":