Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,20 @@ This is useful in large test suites to bound resource usage when many tests
are flaky at the same time. The cap applies after rerun selection, including
tests configured with ``--force-reruns`` and ``@pytest.mark.flaky``.

Limit reruns when exiting early
-------------------------------

By default reruns still happen when ``-x``/``--exitfirst`` or ``--maxfail``
is used, which delays the early exit until a test's reruns are exhausted.
To use a different rerun count in that case, pass ``--reruns-on-exitfirst``:

.. code-block:: bash

$ pytest --reruns 2 -x --reruns-on-exitfirst 0

The value overrides both the global ``--reruns`` setting and per-test
``flaky`` markers, but not ``--force-reruns``.

Show tracebacks for retried failures
------------------------------------

Expand Down
3 changes: 3 additions & 0 deletions changes/249.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Add ``--reruns-on-exitfirst`` option to use a different rerun count when
``-x``/``--exitfirst`` or ``--maxfail`` is given, e.g. ``--reruns-on-exitfirst
0`` to skip reruns entirely when exiting on the first failure.
25 changes: 25 additions & 0 deletions src/pytest_rerunfailures.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,15 @@ def pytest_addoption(parser):
"'rerun test summary info' section, which is emitted automatically "
"when this flag is set.",
)
group._addoption(
"--reruns-on-exitfirst",
action="store",
dest="reruns_on_exitfirst",
type=int,
help="Number of reruns to use when -x/--exitfirst or --maxfail is "
"given. Defaults to the regular rerun settings. Use "
"--reruns-on-exitfirst 0 to skip reruns entirely when exiting early.",
)
group.addoption(
"--max-suite-reruns",
action="store",
Expand Down Expand Up @@ -226,7 +235,16 @@ def check_options(config):
and config.option.max_suite_reruns < 0
):
raise pytest.UsageError("--max-suite-reruns must be >= 0")
if (
config.option.reruns_on_exitfirst is not None
and config.option.reruns_on_exitfirst < 0
):
raise pytest.UsageError("--reruns-on-exitfirst must be >= 0")
reruns = config.getoption("force_reruns") or _get_global_reruns(config)
if not reruns and config.option.maxfail:
# --reruns-on-exitfirst takes effect under -x/--maxfail even when
# no other rerun count is configured
reruns = config.getoption("reruns_on_exitfirst")
if not config.getoption("collectonly") and reruns:
if config.option.usepdb: # a core option
raise pytest.UsageError("--reruns incompatible with --pdb")
Expand All @@ -250,6 +268,13 @@ def get_reruns_count(item):
if reruns is not None:
return reruns

if item.session.config.option.maxfail:
reruns = item.session.config.getoption("reruns_on_exitfirst")
if reruns is not None:
# -x/--exitfirst or --maxfail given: override marker and global
# rerun counts so the run can exit on the first real failure.
return reruns

rerun_marker = _get_marker(item)
# use the marker as a priority over the global setting.
if rerun_marker is not None:
Expand Down
64 changes: 64 additions & 0 deletions tests/test_pytest_rerunfailures.py
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,70 @@ def test_pass():
assert_outcomes(result, passed=1, rerun=2)


def test_reruns_on_exitfirst_zero_disables_reruns(testdir):
testdir.makepyfile("def test_fail(): assert False")
result = testdir.runpytest("--reruns", "2", "-x", "--reruns-on-exitfirst", "0")
assert_outcomes(result, passed=0, failed=1, rerun=0)


def test_reruns_on_exitfirst_overrides_flaky_marker(testdir):
testdir.makepyfile(
f"""
import pytest
@pytest.mark.flaky(reruns=2)
def test_fail():
{temporary_failure()}"""
)
result = testdir.runpytest("-x", "--reruns-on-exitfirst", "0")
assert_outcomes(result, passed=0, failed=1, rerun=0)


def test_reruns_on_exitfirst_limits_reruns(testdir):
testdir.makepyfile(
f"""
def test_fail():
{temporary_failure(3)}"""
)
result = testdir.runpytest("--reruns", "3", "-x", "--reruns-on-exitfirst", "1")
assert_outcomes(result, passed=0, failed=1, rerun=1)


def test_reruns_on_exitfirst_applies_to_maxfail(testdir):
testdir.makepyfile("def test_fail(): assert False")
result = testdir.runpytest(
"--reruns", "2", "--maxfail", "2", "--reruns-on-exitfirst", "0"
)
assert_outcomes(result, passed=0, failed=1, rerun=0)


def test_exitfirst_without_reruns_on_exitfirst_still_reruns(testdir):
testdir.makepyfile(
f"""
def test_fail():
{temporary_failure(2)}"""
)
result = testdir.runpytest("--reruns", "2", "-x")
assert_outcomes(result, passed=1, rerun=2)


def test_reruns_on_exitfirst_negative_rejected(testdir):
testdir.makepyfile("def test_pass(): pass")
result = testdir.runpytest("-x", "--reruns-on-exitfirst", "-1")
result.stderr.fnmatch_lines_random("ERROR: --reruns-on-exitfirst must be >= 0")


def test_error_when_run_with_pdb_and_reruns_on_exitfirst(testdir):
testdir.makepyfile("def test_pass(): pass")
result = testdir.runpytest("-x", "--reruns-on-exitfirst", "1", "--pdb")
result.stderr.fnmatch_lines_random("ERROR: --reruns incompatible with --pdb")


def test_no_error_when_run_with_pdb_and_zero_reruns_on_exitfirst(testdir):
testdir.makepyfile("def test_pass(): pass")
result = testdir.runpytest("-x", "--reruns-on-exitfirst", "0", "--pdb")
assert_outcomes(result)


def test_no_extra_test_summary_for_reruns_by_default(testdir):
testdir.makepyfile(
f"""
Expand Down
Loading