diff --git a/README.rst b/README.rst index f2f18da..4ad2c8e 100644 --- a/README.rst +++ b/README.rst @@ -329,6 +329,7 @@ Here's an example of the output provided by the plugin when run with ============================ rerun test summary info ========================= RERUN test_report.py::test_fail RERUN test_report.py::test_fail + FAILED test_report.py::test_fail ============================ short test summary info ========================= FAIL test_report.py::test_fail ======================= 1 failed, 2 rerun in 0.02 seconds ==================== diff --git a/changes/191.feature.rst b/changes/191.feature.rst new file mode 100644 index 0000000..cf52c03 --- /dev/null +++ b/changes/191.feature.rst @@ -0,0 +1 @@ +Show each rerun attempt and its final outcome in the rerun test summary info section. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index e7dff9b..e86a8db 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -12,6 +12,7 @@ import traceback import warnings from contextlib import suppress +from itertools import chain from typing import Any import pytest @@ -793,6 +794,9 @@ def pytest_handlecrashitem(self, crashitem, report, sched): ) report.longrepr = error_msg + # The attempt index lets the rerun summary order this report + # relative to the rescheduled attempt's own reports. + report.rerun = db.get_test_failures(crashitem) db.add_test_failure(crashitem) @@ -1337,20 +1341,61 @@ def pytest_terminal_summary(terminalreporter): lines = show_rerun(terminalreporter, show_tracebacks=show_tracebacks) if lines: - tr._tw.sep("=", "rerun test summary info") - for line in lines: - tr._tw.line(line) + tr.write_sep("=", "rerun test summary info", cyan=True, bold=True) + for line, markup in lines: + tr.write_line(line, **(markup or {})) def show_rerun(terminalreporter, show_tracebacks=False): + config = terminalreporter.config + attempts = {} + rerun_nodeids = set() + for report in chain.from_iterable(terminalreporter.stats.values()): + if not hasattr(report, "rerun"): + continue + if report.outcome == "rerun": + rerun_nodeids.add(report.nodeid) + attempts.setdefault(report.nodeid, []).append(report) + lines = [] - for rep in terminalreporter.stats.get("rerun", []): - lines.append(f"RERUN {rep.nodeid}") - if show_tracebacks and rep.longrepr: - lines.extend(str(rep.longrepr).splitlines()) + for nodeid, reports in attempts.items(): + if nodeid not in rerun_nodeids: + continue + reports.sort(key=lambda report: (report.rerun, _phase_order(report.when))) + for report in reports: + # A passed setup/teardown report carries no information. + if report.passed and report.when in ("setup", "teardown"): + continue + _, _, word = config.hook.pytest_report_teststatus( + report=report, config=config + ) + if isinstance(word, tuple): + word, markup = word + else: + markup = _outcome_markup(report) + lines.append((f"{word or report.outcome.upper()} {report.nodeid}", markup)) + if show_tracebacks and report.outcome == "rerun" and report.longrepr: + for tb_line in str(report.longrepr).splitlines(): + lines.append((tb_line, None)) return lines +def _phase_order(when): + return {"setup": 0, "call": 1}.get(when, 2) + + +def _outcome_markup(report): + # The default colouring used by pytest's terminal reporter for status + # words returned without explicit markup. + if report.passed and not hasattr(report, "wasxfail"): + return {"green": True} + if report.passed or report.skipped: + return {"yellow": True} + if report.failed: + return {"red": True} + return {} + + @pytest.hookimpl(trylast=True) def pytest_sessionfinish(session, exitstatus): if exitstatus != 0: diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index bc960be..ca8d311 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1,4 +1,5 @@ import random +import re import time from textwrap import indent from types import SimpleNamespace @@ -338,6 +339,38 @@ def test_pass(): ) result = testdir.runpytest("-p", "xdist", "-n", "1", "--reruns", "1", "-r", "R") assert_outcomes(result, passed=2, rerun=1) + stdout = result.stdout.str() + assert ( + "RERUN test_rerun_passes_after_temporary_test_crash.py::test_crash" + in rerun_summary_section(stdout) + ) + + +@pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") +def test_rerun_summary_groups_attempts_by_test(testdir): + testdir.makepyfile( + """ + import py + + def _flaky(name): + path = py.path.local(__file__).dirpath().ensure(name + '.res') + count = int(path.read() or 0) + path.write(count + 1) + assert count >= 1 + + def test_flaky_a(): + _flaky('a') + + def test_flaky_b(): + _flaky('b') + """ + ) + result = testdir.runpytest("-p", "xdist", "-n", "2", "--reruns", "1", "-r", "R") + stdout = result.stdout.str() + # xdist interleaves the reports of both tests; each test's attempts must + # still form a contiguous, attempt-ordered group in the summary. + for name in ("test_flaky_a", "test_flaky_b"): + assert rerun_summary_statuses(stdout, name) == ["RERUN", "PASSED"] @pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") @@ -537,6 +570,129 @@ def test_pass(): assert "1 rerun" in result.stdout.str() +def rerun_summary_section(stdout, keep_colors=False): + """Return the rerun summary section of pytest output.""" + if not keep_colors: + stdout = re.sub(r"\x1b\[[0-9;]*m", "", stdout) + rest = stdout.split("rerun test summary info", 1)[1] + # The section ends at the next separator line (e.g. "short test summary + # info" or the result footer). + lines = rest.splitlines()[1:] + sep = re.compile(r"^[\x1b\[0-9;*m]*=") + end = next((i for i, line in enumerate(lines) if sep.match(line)), len(lines)) + return "\n".join(lines[:end]) + + +def rerun_summary_statuses(stdout, test_name): + section = rerun_summary_section(stdout) + return [ + line.split(maxsplit=1)[0] + for line in section.splitlines() + if f"::{test_name}" in line + ] + + +def test_rerun_summary_shows_each_attempt_outcome(testdir): + testdir.makepyfile( + """ + attempts = 0 + + def test_eventually_passes(): + global attempts + attempts += 1 + assert attempts == 3 + """ + ) + result = testdir.runpytest("--reruns", "2", "-r", "R", "--color", "yes") + + stdout = result.stdout.str() + # Scope the colour assertions to the rerun summary section: with -v the + # progress line already contains a green PASSED. + section = rerun_summary_section(stdout, keep_colors=True) + assert "\x1b[33mRERUN " in section + assert "\x1b[32mPASSED " in section + assert rerun_summary_statuses(stdout, "test_eventually_passes") == [ + "RERUN", + "RERUN", + "PASSED", + ] + + +def test_rerun_summary_shows_call_and_teardown_failures(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def bad_teardown(): + yield + raise RuntimeError("teardown exploded") + + def test_fails(bad_teardown): + assert False, "call exploded" + """ + ) + result = testdir.runpytest("--reruns", "1", "--rerun-show-tracebacks") + + stdout = result.stdout.str() + assert rerun_summary_statuses(stdout, "test_fails") == [ + "RERUN", + "RERUN", + "FAILED", + "ERROR", + ] + section = rerun_summary_section(stdout) + assert "call exploded" in section + # Both rerun reports carry their traceback, including the teardown one. + assert "teardown exploded" in section + + +def test_rerun_summary_shows_setup_error(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def broken(): + raise RuntimeError("setup exploded") + + def test_needs_fixture(broken): + pass + """ + ) + result = testdir.runpytest("--reruns", "1", "-r", "R") + + # A failed setup produces no call report, so each attempt has one line. + assert rerun_summary_statuses(result.stdout.str(), "test_needs_fixture") == [ + "RERUN", + "ERROR", + ] + + +def test_rerun_summary_shows_skipped_call(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def bad_teardown(): + yield + raise RuntimeError("teardown exploded") + + def test_skips(bad_teardown): + pytest.skip("not today") + """ + ) + result = testdir.runpytest("--reruns", "1", "-r", "R") + + assert rerun_summary_statuses(result.stdout.str(), "test_skips") == [ + "SKIPPED", + "RERUN", + "SKIPPED", + "ERROR", + ] + + def test_rerun_show_tracebacks_for_eventual_pass(testdir): testdir.makepyfile( f"""