Add --rerun-warning flag to emit a warning on each scheduled rerun

README.rst

@@ -334,6 +334,21 @@ Each retried attempt's traceback is appended to the ``rerun test summary
 info`` section. The section is emitted automatically when the flag is set,
 so ``-rR`` is not required.
 
+Warn when a test is rerun
+-------------------------
+
+To make reruns more visible on CI, pass ``--rerun-warning``. Each time a
+test is scheduled for rerun a ``PytestWarning`` is emitted, which tools
+like `pytest-github-actions-annotate-failures
+<https://pypi.org/project/pytest-github-actions-annotate-failures/>`_ can
+turn into annotations:
+
+.. code-block:: bash
+
+   $ pytest --reruns 2 --rerun-warning
+
+Note that ``filterwarnings = error`` turns these warnings into errors.
+
 Output
 ------
 

changes/318.feature.rst

@@ -0,0 +1,3 @@
+Add ``--rerun-warning`` flag that emits a ``PytestWarning`` each time a test
+is scheduled for rerun, to surface flaky tests in CI (e.g. via
+pytest-github-actions-annotate-failures annotations).

src/pytest_rerunfailures.py

@@ -178,6 +178,15 @@ def pytest_addoption(parser):
         "'rerun test summary info' section, which is emitted automatically "
         "when this flag is set.",
     )
+    group._addoption(
+        "--rerun-warning",
+        action="store_true",
+        dest="rerun_warning",
+        help="Emit a PytestWarning each time a test is scheduled for rerun. "
+        "Useful to surface flaky tests in CI, e.g. via annotations from "
+        "pytest-github-actions-annotate-failures. Note that "
+        "filterwarnings=error turns these warnings into errors.",
+    )
     group.addoption(
         "--max-suite-reruns",
         action="store",
@@ -1337,6 +1346,13 @@ def pytest_runtest_protocol(item, nextitem):
                         continue
 
                 report.outcome = "rerun"
+                if item.session.config.option.rerun_warning:
+                    item.warn(
+                        pytest.PytestWarning(
+                            f"{item.nodeid} failed on attempt "
+                            f"{item.execution_count} and will be rerun"
+                        )
+                    )
                 time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1))
 
                 if not parallel or works_with_current_xdist():

tests/test_pytest_rerunfailures.py

@@ -798,6 +798,29 @@ def test_skips(bad_teardown):
     ]
 
 
+def test_rerun_warning_emitted(testdir):
+    testdir.makepyfile(
+        f"""
+        def test_pass():
+            {temporary_failure()}"""
+    )
+    result = testdir.runpytest("--reruns", "1", "--rerun-warning")
+    assert_outcomes(result, passed=1, rerun=1)
+    result.stdout.fnmatch_lines_random([
+        "*PytestWarning: *test_pass* failed on attempt 1 and will be rerun*"
+    ])
+
+
+def test_no_rerun_warning_by_default(testdir):
+    testdir.makepyfile(
+        f"""
+        def test_pass():
+            {temporary_failure()}"""
+    )
+    result = testdir.runpytest("--reruns", "1")
+    assert "will be rerun" not in result.stdout.str()
+
+
 def test_rerun_show_tracebacks_for_eventual_pass(testdir):
     testdir.makepyfile(
         f"""