From 0b020215d6f8ede191b9f7a5b4f8e9fdbb862701 Mon Sep 17 00:00:00 2001 From: punithk-verse Date: Tue, 15 Sep 2026 12:28:18 +0000 Subject: [PATCH 1/2] fix: avoid counting Go subtests as untracked failures --- .../pipelines/_pr_runtime_verifier.py | 31 ++++++-- tests/test_pr_runtime_verifier.py | 72 +++++++++++++++++++ 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/repo2rlenv/pipelines/_pr_runtime_verifier.py b/src/repo2rlenv/pipelines/_pr_runtime_verifier.py index b5bb2352..84b871f4 100644 --- a/src/repo2rlenv/pipelines/_pr_runtime_verifier.py +++ b/src/repo2rlenv/pipelines/_pr_runtime_verifier.py @@ -238,12 +238,23 @@ def parse_logs(runner: str, log: str) -> dict[str, str]: # --------------------------------------------------------------------------- # Grading # --------------------------------------------------------------------------- +def _has_go_parent(name: str, status_map: dict[str, str]) -> bool: + """Return True if a Go test has an ancestor represented in the status map.""" + parts = name.split("/") + + for i in range(1, len(parts)): + parent = "/".join(parts[:i]) + if parent in status_map: + return True + + return False def grade( fail_to_pass: list[str], pass_to_pass: list[str], status_map: dict[str, str], + runner: str | None = None, ) -> dict: """Compute the graded reward + strict resolved bool from a status map. @@ -276,10 +287,22 @@ def grade( p2p_passed = sum(1 for t in pass_to_pass if status_map.get(t) == PASSED) # Tests that should have stayed green but regressed (PASS->not-pass). regressions = [t for t in pass_to_pass if status_map.get(t) != PASSED] + # FAILED tests outside the tracked sets — the selected command isn't clean. - untracked_failed = sorted( - t for t, s in status_map.items() if s == FAILED and t not in f2p_set and t not in p2p_set - ) + if runner == "go": + failed_tests = { + t + for t, s in status_map.items() + if s == FAILED and t not in f2p_set and t not in p2p_set + } + + untracked_failed = sorted(t for t in failed_tests if not _has_go_parent(t, status_map)) + else: + untracked_failed = sorted( + t + for t, s in status_map.items() + if s == FAILED and t not in f2p_set and t not in p2p_set + ) f2p_rate = (f2p_passed / f2p_total) if f2p_total else 0.0 p2p_rate = (p2p_passed / p2p_total) if p2p_total else 1.0 @@ -364,7 +387,7 @@ def main(argv: list[str] | None = None) -> int: "exit_code": args.exit_code, } else: - breakdown = grade(f2p, p2p, status_map) + breakdown = grade(f2p, p2p, status_map, runner) breakdown["parse_status"] = "ok" breakdown["runner"] = runner breakdown["tests_parsed"] = len(status_map) diff --git a/tests/test_pr_runtime_verifier.py b/tests/test_pr_runtime_verifier.py index d134cc24..a4c88386 100644 --- a/tests/test_pr_runtime_verifier.py +++ b/tests/test_pr_runtime_verifier.py @@ -130,6 +130,40 @@ def test_grade_untracked_failure_keeps_tracked_resolved(): assert r["untracked_failed"] == ["tests/other::cp1252"] +def test_grade_go_subtests_do_not_count_when_parent_passes(): + """Go subtest failures under a passing parent do not block command resolution.""" + r = grade( + [], + [], + { + "TestExample": "PASSED", + "TestExample/subtest1": "FAILED", + "TestExample/subtest2": "FAILED", + }, + runner="go", + ) + + assert r["untracked_failed_count"] == 0 + assert r["untracked_failed"] == [] + + +def test_grade_go_subtests_count_parent_failure_once(): + """Go subtests under a failed parent are represented by the parent failure.""" + r = grade( + [], + [], + { + "TestExample": "FAILED", + "TestExample/subtest1": "FAILED", + "TestExample/subtest2": "FAILED", + }, + runner="go", + ) + + assert r["untracked_failed_count"] == 1 + assert r["untracked_failed"] == ["TestExample"] + + def test_grade_no_untracked_failure_resolves(): r = grade(["f1"], ["keep"], {"f1": "PASSED", "keep": "PASSED"}) assert r["resolved"] is True @@ -178,6 +212,44 @@ def test_main_writes_graded_reward(tmp_path: Path): assert breakdown["exit_code"] == 0 # always recorded, not just in fallback +def test_main_go_subtest_failure_under_passing_parent_is_not_untracked(tmp_path: Path): + """Go subtest failures under a passing parent should not be counted separately.""" + log = _write( + tmp_path / "out.log", + """--- PASS: TestTracked (0.01s) + --- PASS: TestExample (0.01s) + --- FAIL: TestExample/subtest1 (0.00s) + --- FAIL: TestExample/subtest2 (0.00s) + """, + ) + f2p = _write(tmp_path / "f2p.json", json.dumps(["TestTracked"])) + p2p = _write(tmp_path / "p2p.json", json.dumps([])) + out_dir = tmp_path / "verifier" + + main( + [ + "--log", + log, + "--f2p", + f2p, + "--p2p", + p2p, + "--runner", + "go", + "--exit-code", + "0", + "--out-dir", + str(out_dir), + ] + ) + + breakdown = json.loads((out_dir / "reward-details.json").read_text()) + assert breakdown["f2p_passed"] == 1 + assert breakdown["untracked_failed_count"] == 0 + assert breakdown["untracked_failed"] == [] + assert breakdown["command_resolved"] is True + + def test_main_command_resolved_false_on_untracked_failure(tmp_path: Path): """All tracked tests pass (resolved True) but an untracked test failed and the command exited nonzero -> resolved True, command_resolved False.""" From 501902e17fc317a9823ec62cc9187405d30aa6e2 Mon Sep 17 00:00:00 2001 From: punithk-verse Date: Thu, 17 Sep 2026 10:50:04 +0000 Subject: [PATCH 2/2] fix: only group Go subtests under failed parents --- .../pipelines/_pr_runtime_verifier.py | 2 +- tests/test_pr_runtime_verifier.py | 24 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/repo2rlenv/pipelines/_pr_runtime_verifier.py b/src/repo2rlenv/pipelines/_pr_runtime_verifier.py index 84b871f4..a113ed37 100644 --- a/src/repo2rlenv/pipelines/_pr_runtime_verifier.py +++ b/src/repo2rlenv/pipelines/_pr_runtime_verifier.py @@ -244,7 +244,7 @@ def _has_go_parent(name: str, status_map: dict[str, str]) -> bool: for i in range(1, len(parts)): parent = "/".join(parts[:i]) - if parent in status_map: + if status_map.get(parent) == FAILED: return True return False diff --git a/tests/test_pr_runtime_verifier.py b/tests/test_pr_runtime_verifier.py index a4c88386..fef651e6 100644 --- a/tests/test_pr_runtime_verifier.py +++ b/tests/test_pr_runtime_verifier.py @@ -131,7 +131,7 @@ def test_grade_untracked_failure_keeps_tracked_resolved(): def test_grade_go_subtests_do_not_count_when_parent_passes(): - """Go subtest failures under a passing parent do not block command resolution.""" + """Go subtest failures under a passing parent remain untracked.""" r = grade( [], [], @@ -142,10 +142,11 @@ def test_grade_go_subtests_do_not_count_when_parent_passes(): }, runner="go", ) - - assert r["untracked_failed_count"] == 0 - assert r["untracked_failed"] == [] - + assert r["untracked_failed_count"] == 2 + assert r["untracked_failed"] == [ + "TestExample/subtest1", + "TestExample/subtest2", + ] def test_grade_go_subtests_count_parent_failure_once(): """Go subtests under a failed parent are represented by the parent failure.""" @@ -212,8 +213,8 @@ def test_main_writes_graded_reward(tmp_path: Path): assert breakdown["exit_code"] == 0 # always recorded, not just in fallback -def test_main_go_subtest_failure_under_passing_parent_is_not_untracked(tmp_path: Path): - """Go subtest failures under a passing parent should not be counted separately.""" +def test_main_go_subtest_failure_under_passing_parent_is_untracked(tmp_path: Path): + """Go subtest failures under a passing parent remain untracked.""" log = _write( tmp_path / "out.log", """--- PASS: TestTracked (0.01s) @@ -245,9 +246,12 @@ def test_main_go_subtest_failure_under_passing_parent_is_not_untracked(tmp_path: breakdown = json.loads((out_dir / "reward-details.json").read_text()) assert breakdown["f2p_passed"] == 1 - assert breakdown["untracked_failed_count"] == 0 - assert breakdown["untracked_failed"] == [] - assert breakdown["command_resolved"] is True + assert breakdown["untracked_failed_count"] == 2 + assert breakdown["untracked_failed"] == [ + "TestExample/subtest1", + "TestExample/subtest2", + ] + assert breakdown["command_resolved"] is False def test_main_command_resolved_false_on_untracked_failure(tmp_path: Path):