From c24794676bd6f0a8ec939f5629643486ca65905f Mon Sep 17 00:00:00 2001 From: Xiang Chucheng Date: Wed, 16 Sep 2026 22:13:30 +0800 Subject: [PATCH 1/2] fix(swe-smith): parse pytest statuses that include ANSI color Pytest color codes glue themselves to PASSED, so the reward parser misses every passing node and scores a green suite as 0/N. Co-authored-by: Cursor --- examples/swe_smith/agents/smith_agent.py | 8 +++-- tests/examples/test_swe_smith_agent.py | 37 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/examples/swe_smith/agents/smith_agent.py b/examples/swe_smith/agents/smith_agent.py index 94aebe795..aff033634 100755 --- a/examples/swe_smith/agents/smith_agent.py +++ b/examples/swe_smith/agents/smith_agent.py @@ -31,6 +31,10 @@ _STATUS_RE = re.compile( r"(?:^|\s)(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)\s+(\S+)|(\S+)\s+(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)(?:\s|$)" ) +# pytest --color=yes (or a repo that forces color) wraps PASSED in ANSI codes. +# Without stripping them, parse_test_statuses never sees a bare "PASSED" token +# and a fully passing suite is scored 0/N. +_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") SUBMIT_MARKER = "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT" @@ -571,7 +575,7 @@ def capture_patch() -> str: def parse_test_statuses(test_output: str) -> dict[str, str]: statuses: dict[str, str] = {} - for line in test_output.splitlines(): + for line in _ANSI_ESCAPE_RE.sub("", test_output).splitlines(): m = _STATUS_RE.search(line) if not m: continue @@ -603,7 +607,7 @@ def evaluate(eval_meta: dict[str, Any], timeout: int, f2p_only: bool = True) -> # needs xdist, so probe first and fall back to serial `-p no:xdist` when absent. xdist_flag = "-n4" if _run("python -c 'import xdist'", 30)[1] == 0 else "-p no:xdist" output, rc = _run( - "python -m pytest -rA -p no:cacheprovider " + xdist_flag + " " + " ".join(map(_shq, nodes)), + "python -m pytest --color=no -rA -p no:cacheprovider " + xdist_flag + " " + " ".join(map(_shq, nodes)), timeout, ) # rc 124 == subprocess.TimeoutExpired (see _run); pytest itself never exits 124. diff --git a/tests/examples/test_swe_smith_agent.py b/tests/examples/test_swe_smith_agent.py index c79a27e0b..1f30112d6 100644 --- a/tests/examples/test_swe_smith_agent.py +++ b/tests/examples/test_swe_smith_agent.py @@ -58,6 +58,43 @@ class _Chat: self.chat = _Chat() +def test_parse_test_statuses_strips_ansi_color_codes() -> None: + output = ( + "\x1b[32mPASSED\x1b[0m tests/test_database.py::\x1b[1mCanToolsDatabaseTest::test_issue_62\x1b[0m\n" + "FAILED tests/test_other.py::test_broken\n" + ) + + statuses = smith_agent.parse_test_statuses(output) + + assert statuses["tests/test_database.py::CanToolsDatabaseTest::test_issue_62"] == "PASSED" + assert statuses["tests/test_other.py::test_broken"] == "FAILED" + + +def test_evaluate_disables_pytest_color(monkeypatch) -> None: + commands: list[str] = [] + + def fake_run(command: str, timeout: int) -> tuple[str, int]: + commands.append(command) + if "import xdist" in command: + return "", 1 + return "PASSED tests/test_demo.py::test_ok\n", 0 + + monkeypatch.setattr(smith_agent, "restore_f2p_tests", lambda *_args, **_kwargs: None) + monkeypatch.setattr(smith_agent, "_run", fake_run) + + reward, resolved, reason, timed_out = smith_agent.evaluate( + {"FAIL_TO_PASS": ["tests/test_demo.py::test_ok"], "PASS_TO_PASS": []}, + timeout=30, + ) + + pytest_commands = [command for command in commands if "python -m pytest" in command] + assert pytest_commands and "--color=no" in pytest_commands[0] + assert reward == 1.0 + assert resolved is True + assert timed_out is False + assert "1/1" in reason + + def test_context_overflow_400_ends_loop_without_empty_turn() -> None: client = _RaisingClient(_bad_request(_OVERFLOW_MESSAGE)) From f175d0594747e811ef71adc6d79f65e0616eb32c Mon Sep 17 00:00:00 2001 From: Xiang Chucheng Date: Thu, 17 Sep 2026 16:56:12 +0800 Subject: [PATCH 2/2] fix(swe-smith): drop extra ANSI parser tests per review Co-authored-by: Cursor --- tests/examples/test_swe_smith_agent.py | 37 -------------------------- 1 file changed, 37 deletions(-) diff --git a/tests/examples/test_swe_smith_agent.py b/tests/examples/test_swe_smith_agent.py index 1f30112d6..c79a27e0b 100644 --- a/tests/examples/test_swe_smith_agent.py +++ b/tests/examples/test_swe_smith_agent.py @@ -58,43 +58,6 @@ class _Chat: self.chat = _Chat() -def test_parse_test_statuses_strips_ansi_color_codes() -> None: - output = ( - "\x1b[32mPASSED\x1b[0m tests/test_database.py::\x1b[1mCanToolsDatabaseTest::test_issue_62\x1b[0m\n" - "FAILED tests/test_other.py::test_broken\n" - ) - - statuses = smith_agent.parse_test_statuses(output) - - assert statuses["tests/test_database.py::CanToolsDatabaseTest::test_issue_62"] == "PASSED" - assert statuses["tests/test_other.py::test_broken"] == "FAILED" - - -def test_evaluate_disables_pytest_color(monkeypatch) -> None: - commands: list[str] = [] - - def fake_run(command: str, timeout: int) -> tuple[str, int]: - commands.append(command) - if "import xdist" in command: - return "", 1 - return "PASSED tests/test_demo.py::test_ok\n", 0 - - monkeypatch.setattr(smith_agent, "restore_f2p_tests", lambda *_args, **_kwargs: None) - monkeypatch.setattr(smith_agent, "_run", fake_run) - - reward, resolved, reason, timed_out = smith_agent.evaluate( - {"FAIL_TO_PASS": ["tests/test_demo.py::test_ok"], "PASS_TO_PASS": []}, - timeout=30, - ) - - pytest_commands = [command for command in commands if "python -m pytest" in command] - assert pytest_commands and "--color=no" in pytest_commands[0] - assert reward == 1.0 - assert resolved is True - assert timed_out is False - assert "1/1" in reason - - def test_context_overflow_400_ends_loop_without_empty_turn() -> None: client = _RaisingClient(_bad_request(_OVERFLOW_MESSAGE))