diff --git a/README.rst b/README.rst index 747072ee..a57c1958 100644 --- a/README.rst +++ b/README.rst @@ -304,6 +304,20 @@ This is useful in large test suites to bound resource usage when many tests are flaky at the same time. The cap applies after rerun selection, including tests configured with ``--force-reruns`` and ``@pytest.mark.flaky``. +Mark exhausted flaky failures as xfail +-------------------------------------- + +To keep flaky failures from failing the test run, pass ``--xfail-flaky``. +A test that still fails after all its re-runs is then reported as +``xfailed`` instead of ``failed``: + +.. code-block:: bash + + $ pytest --reruns 2 --xfail-flaky + +Only tests that were actually re-run are affected. Failures excluded from +re-running by ``--only-rerun``/``--rerun-except`` still report as failed. + Show tracebacks for retried failures ------------------------------------ diff --git a/changes/92.feature.rst b/changes/92.feature.rst new file mode 100644 index 00000000..8f283c21 --- /dev/null +++ b/changes/92.feature.rst @@ -0,0 +1,2 @@ +Add ``--xfail-flaky`` flag that reports a test as ``xfailed`` instead of +``failed`` when it still fails after all its re-runs are exhausted. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 616e658b..900f20fc 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -169,6 +169,14 @@ def pytest_addoption(parser): dest="fail_on_flaky", help="Fail the test run with exit code 7 if a flaky test passes on a rerun.", ) + group._addoption( + "--xfail-flaky", + action="store_true", + dest="xfail_flaky", + help="Mark tests that still fail after exhausting their reruns as " + "xfailed instead of failed. Only applies to tests that were actually " + "rerun, and not to failures excluded by --only-rerun/--rerun-except.", + ) group._addoption( "--rerun-show-tracebacks", action="store_true", @@ -1285,6 +1293,19 @@ def pytest_runtest_protocol(item, nextitem): # nonmatching failure as a final result first. continue elif _should_not_rerun(item, report, reruns, condition): + if ( + item.session.config.option.xfail_flaky + and report.failed + and item.execution_count > reruns + and item.execution_count > 1 + and not any(item._terminal_errors.values()) + ): + # reruns are exhausted: report the final failure as xfail + report.outcome = "skipped" + report.wasxfail = ( + f"test failed after {item.execution_count - 1} " + "rerun(s), marked as xfail" + ) # no rerun needed or one already triggered, log normally item.ihook.pytest_runtest_logreport(report=report) else: diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index a4208c50..a8509b7d 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -548,6 +548,68 @@ def test_pass(): assert_outcomes(result, passed=1, rerun=2) +def test_xfail_flaky_marks_exhausted_reruns_as_xfail(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--reruns", "2", "--xfail-flaky") + assert result.ret == 0 + assert_outcomes(result, passed=0, failed=0, xfailed=1, rerun=2) + + +def test_xfail_flaky_does_not_affect_eventual_pass(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()}""" + ) + result = testdir.runpytest("--reruns", "2", "--xfail-flaky") + assert_outcomes(result, passed=1, rerun=1) + + +def test_xfail_flaky_does_not_affect_test_without_reruns(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--xfail-flaky") + assert_outcomes(result, passed=0, failed=1, xfailed=0, rerun=0) + + +def test_xfail_flaky_ignores_unmatched_only_rerun_error(testdir): + testdir.makepyfile("def test_fail(): raise ValueError('nope')") + result = testdir.runpytest( + "--reruns", "2", "--only-rerun", "AssertionError", "--xfail-flaky" + ) + assert_outcomes(result, passed=0, failed=1, xfailed=0, rerun=0) + + +def test_xfail_flaky_with_flaky_marker(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.mark.flaky(reruns=1) + def test_fail(): assert False + """ + ) + result = testdir.runpytest("--xfail-flaky") + assert result.ret == 0 + assert_outcomes(result, passed=0, failed=0, xfailed=1, rerun=1) + + +def test_xfail_flaky_marks_exhausted_setup_failures_as_xfail(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def broken_fixture(): + raise ValueError("setup always fails") + + def test_fail(broken_fixture): pass + """ + ) + result = testdir.runpytest("--reruns", "2", "--xfail-flaky") + assert result.ret == 0 + assert_outcomes(result, passed=0, failed=0, xfailed=1, rerun=2) + + def test_no_extra_test_summary_for_reruns_by_default(testdir): testdir.makepyfile( f"""