From 80ae22fb3e39206b4207ceb664ab60a4775674e4 Mon Sep 17 00:00:00 2001 From: Louis Deconinck Date: Thu, 17 Sep 2026 21:33:58 +0200 Subject: [PATCH 1/2] Add --xfail-flaky to report exhausted flaky failures as xfail A test that still fails after all its reruns are exhausted is reported as xfailed instead of failed, similar to pytest-ignore-flaky. Only tests that actually reran are affected, and failures excluded by --only-rerun/--rerun-except still fail. Closes #92. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.rst | 14 ++++++++++ changes/92.feature.rst | 2 ++ src/pytest_rerunfailures.py | 22 +++++++++++++++ tests/test_pytest_rerunfailures.py | 45 ++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 changes/92.feature.rst diff --git a/README.rst b/README.rst index 747072e..a57c195 100644 --- a/README.rst +++ b/README.rst @@ -304,6 +304,20 @@ This is useful in large test suites to bound resource usage when many tests are flaky at the same time. The cap applies after rerun selection, including tests configured with ``--force-reruns`` and ``@pytest.mark.flaky``. +Mark exhausted flaky failures as xfail +-------------------------------------- + +To keep flaky failures from failing the test run, pass ``--xfail-flaky``. +A test that still fails after all its re-runs is then reported as +``xfailed`` instead of ``failed``: + +.. code-block:: bash + + $ pytest --reruns 2 --xfail-flaky + +Only tests that were actually re-run are affected. Failures excluded from +re-running by ``--only-rerun``/``--rerun-except`` still report as failed. + Show tracebacks for retried failures ------------------------------------ diff --git a/changes/92.feature.rst b/changes/92.feature.rst new file mode 100644 index 0000000..8f283c2 --- /dev/null +++ b/changes/92.feature.rst @@ -0,0 +1,2 @@ +Add ``--xfail-flaky`` flag that reports a test as ``xfailed`` instead of +``failed`` when it still fails after all its re-runs are exhausted. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 616e658..3d6890f 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -169,6 +169,14 @@ def pytest_addoption(parser): dest="fail_on_flaky", help="Fail the test run with exit code 7 if a flaky test passes on a rerun.", ) + group._addoption( + "--xfail-flaky", + action="store_true", + dest="xfail_flaky", + help="Mark tests that still fail after exhausting their reruns as " + "xfailed instead of failed. Only applies to tests that were actually " + "rerun, and not to failures excluded by --only-rerun/--rerun-except.", + ) group._addoption( "--rerun-show-tracebacks", action="store_true", @@ -1285,6 +1293,20 @@ def pytest_runtest_protocol(item, nextitem): # nonmatching failure as a final result first. continue elif _should_not_rerun(item, report, reruns, condition): + if ( + item.session.config.option.xfail_flaky + and report.when == "call" + and report.failed + and item.execution_count > reruns + and item.execution_count > 1 + and not any(item._terminal_errors.values()) + ): + # reruns are exhausted: report the final failure as xfail + report.outcome = "skipped" + report.wasxfail = ( + f"test failed after {item.execution_count - 1} " + "rerun(s), marked as xfail" + ) # no rerun needed or one already triggered, log normally item.ihook.pytest_runtest_logreport(report=report) else: diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index a4208c5..c53bd30 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -548,6 +548,51 @@ def test_pass(): assert_outcomes(result, passed=1, rerun=2) +def test_xfail_flaky_marks_exhausted_reruns_as_xfail(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--reruns", "2", "--xfail-flaky") + assert result.ret == 0 + assert_outcomes(result, passed=0, failed=0, xfailed=1, rerun=2) + + +def test_xfail_flaky_does_not_affect_eventual_pass(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()}""" + ) + result = testdir.runpytest("--reruns", "2", "--xfail-flaky") + assert_outcomes(result, passed=1, rerun=1) + + +def test_xfail_flaky_does_not_affect_test_without_reruns(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--xfail-flaky") + assert_outcomes(result, passed=0, failed=1, xfailed=0, rerun=0) + + +def test_xfail_flaky_ignores_unmatched_only_rerun_error(testdir): + testdir.makepyfile("def test_fail(): raise ValueError('nope')") + result = testdir.runpytest( + "--reruns", "2", "--only-rerun", "AssertionError", "--xfail-flaky" + ) + assert_outcomes(result, passed=0, failed=1, xfailed=0, rerun=0) + + +def test_xfail_flaky_with_flaky_marker(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.mark.flaky(reruns=1) + def test_fail(): assert False + """ + ) + result = testdir.runpytest("--xfail-flaky") + assert result.ret == 0 + assert_outcomes(result, passed=0, failed=0, xfailed=1, rerun=1) + + def test_no_extra_test_summary_for_reruns_by_default(testdir): testdir.makepyfile( f""" From 53b61ecbbdac4319053863f6088415d81f2cef87 Mon Sep 17 00:00:00 2001 From: Louis Deconinck Date: Fri, 18 Sep 2026 08:17:18 +0200 Subject: [PATCH 2/2] Also xfail exhausted setup/teardown failures with --xfail-flaky The conversion only applied to call-phase reports, so a fixture that failed on every attempt still finished as a normal failure. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/pytest_rerunfailures.py | 1 - tests/test_pytest_rerunfailures.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 3d6890f..900f20f 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -1295,7 +1295,6 @@ def pytest_runtest_protocol(item, nextitem): elif _should_not_rerun(item, report, reruns, condition): if ( item.session.config.option.xfail_flaky - and report.when == "call" and report.failed and item.execution_count > reruns and item.execution_count > 1 diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index c53bd30..a8509b7 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -593,6 +593,23 @@ def test_fail(): assert False assert_outcomes(result, passed=0, failed=0, xfailed=1, rerun=1) +def test_xfail_flaky_marks_exhausted_setup_failures_as_xfail(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def broken_fixture(): + raise ValueError("setup always fails") + + def test_fail(broken_fixture): pass + """ + ) + result = testdir.runpytest("--reruns", "2", "--xfail-flaky") + assert result.ret == 0 + assert_outcomes(result, passed=0, failed=0, xfailed=1, rerun=2) + + def test_no_extra_test_summary_for_reruns_by_default(testdir): testdir.makepyfile( f"""