diff --git a/src/repo2rlenv/pipelines/_pr_runtime_verifier.py b/src/repo2rlenv/pipelines/_pr_runtime_verifier.py index b5bb235..a113ed3 100644 --- a/src/repo2rlenv/pipelines/_pr_runtime_verifier.py +++ b/src/repo2rlenv/pipelines/_pr_runtime_verifier.py @@ -238,12 +238,23 @@ def parse_logs(runner: str, log: str) -> dict[str, str]: # --------------------------------------------------------------------------- # Grading # --------------------------------------------------------------------------- +def _has_go_parent(name: str, status_map: dict[str, str]) -> bool: + """Return True if a Go test has an ancestor represented in the status map.""" + parts = name.split("/") + + for i in range(1, len(parts)): + parent = "/".join(parts[:i]) + if status_map.get(parent) == FAILED: + return True + + return False def grade( fail_to_pass: list[str], pass_to_pass: list[str], status_map: dict[str, str], + runner: str | None = None, ) -> dict: """Compute the graded reward + strict resolved bool from a status map. @@ -276,10 +287,22 @@ def grade( p2p_passed = sum(1 for t in pass_to_pass if status_map.get(t) == PASSED) # Tests that should have stayed green but regressed (PASS->not-pass). regressions = [t for t in pass_to_pass if status_map.get(t) != PASSED] + # FAILED tests outside the tracked sets — the selected command isn't clean. - untracked_failed = sorted( - t for t, s in status_map.items() if s == FAILED and t not in f2p_set and t not in p2p_set - ) + if runner == "go": + failed_tests = { + t + for t, s in status_map.items() + if s == FAILED and t not in f2p_set and t not in p2p_set + } + + untracked_failed = sorted(t for t in failed_tests if not _has_go_parent(t, status_map)) + else: + untracked_failed = sorted( + t + for t, s in status_map.items() + if s == FAILED and t not in f2p_set and t not in p2p_set + ) f2p_rate = (f2p_passed / f2p_total) if f2p_total else 0.0 p2p_rate = (p2p_passed / p2p_total) if p2p_total else 1.0 @@ -364,7 +387,7 @@ def main(argv: list[str] | None = None) -> int: "exit_code": args.exit_code, } else: - breakdown = grade(f2p, p2p, status_map) + breakdown = grade(f2p, p2p, status_map, runner) breakdown["parse_status"] = "ok" breakdown["runner"] = runner breakdown["tests_parsed"] = len(status_map) diff --git a/tests/test_pr_runtime_verifier.py b/tests/test_pr_runtime_verifier.py index d134cc2..fef651e 100644 --- a/tests/test_pr_runtime_verifier.py +++ b/tests/test_pr_runtime_verifier.py @@ -130,6 +130,41 @@ def test_grade_untracked_failure_keeps_tracked_resolved(): assert r["untracked_failed"] == ["tests/other::cp1252"] +def test_grade_go_subtests_do_not_count_when_parent_passes(): + """Go subtest failures under a passing parent remain untracked.""" + r = grade( + [], + [], + { + "TestExample": "PASSED", + "TestExample/subtest1": "FAILED", + "TestExample/subtest2": "FAILED", + }, + runner="go", + ) + assert r["untracked_failed_count"] == 2 + assert r["untracked_failed"] == [ + "TestExample/subtest1", + "TestExample/subtest2", + ] + +def test_grade_go_subtests_count_parent_failure_once(): + """Go subtests under a failed parent are represented by the parent failure.""" + r = grade( + [], + [], + { + "TestExample": "FAILED", + "TestExample/subtest1": "FAILED", + "TestExample/subtest2": "FAILED", + }, + runner="go", + ) + + assert r["untracked_failed_count"] == 1 + assert r["untracked_failed"] == ["TestExample"] + + def test_grade_no_untracked_failure_resolves(): r = grade(["f1"], ["keep"], {"f1": "PASSED", "keep": "PASSED"}) assert r["resolved"] is True @@ -178,6 +213,47 @@ def test_main_writes_graded_reward(tmp_path: Path): assert breakdown["exit_code"] == 0 # always recorded, not just in fallback +def test_main_go_subtest_failure_under_passing_parent_is_untracked(tmp_path: Path): + """Go subtest failures under a passing parent remain untracked.""" + log = _write( + tmp_path / "out.log", + """--- PASS: TestTracked (0.01s) + --- PASS: TestExample (0.01s) + --- FAIL: TestExample/subtest1 (0.00s) + --- FAIL: TestExample/subtest2 (0.00s) + """, + ) + f2p = _write(tmp_path / "f2p.json", json.dumps(["TestTracked"])) + p2p = _write(tmp_path / "p2p.json", json.dumps([])) + out_dir = tmp_path / "verifier" + + main( + [ + "--log", + log, + "--f2p", + f2p, + "--p2p", + p2p, + "--runner", + "go", + "--exit-code", + "0", + "--out-dir", + str(out_dir), + ] + ) + + breakdown = json.loads((out_dir / "reward-details.json").read_text()) + assert breakdown["f2p_passed"] == 1 + assert breakdown["untracked_failed_count"] == 2 + assert breakdown["untracked_failed"] == [ + "TestExample/subtest1", + "TestExample/subtest2", + ] + assert breakdown["command_resolved"] is False + + def test_main_command_resolved_false_on_untracked_failure(tmp_path: Path): """All tracked tests pass (resolved True) but an untracked test failed and the command exited nonzero -> resolved True, command_resolved False."""