From 33d6a799ad90699585561345307e9d3c7dd08e72 Mon Sep 17 00:00:00 2001 From: David Bednarczyk Date: Mon, 12 Oct 2026 18:56:39 -0400 Subject: [PATCH] Evals: an API judge gets findings too long to paste as its judge files --- .../vo-t01/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-t02/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-t03/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-t04/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-t05/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-t06/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-x1/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-x17/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-x20/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-x4/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-x7/steps/validator/tests/finish.py | 16 ++++++++++++++-- .../vo-x8/steps/validator/tests/finish.py | 16 ++++++++++++++-- 12 files changed, 168 insertions(+), 24 deletions(-) diff --git a/evals/team-tasks/vo-t01/steps/validator/tests/finish.py b/evals/team-tasks/vo-t01/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t01/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t01/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t02/steps/validator/tests/finish.py b/evals/team-tasks/vo-t02/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t02/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t02/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t03/steps/validator/tests/finish.py b/evals/team-tasks/vo-t03/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t03/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t03/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t04/steps/validator/tests/finish.py b/evals/team-tasks/vo-t04/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t04/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t04/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t05/steps/validator/tests/finish.py b/evals/team-tasks/vo-t05/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t05/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t05/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t06/steps/validator/tests/finish.py b/evals/team-tasks/vo-t06/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t06/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t06/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x1/steps/validator/tests/finish.py b/evals/team-tasks/vo-x1/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x1/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x1/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x17/steps/validator/tests/finish.py b/evals/team-tasks/vo-x17/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x17/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x17/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x20/steps/validator/tests/finish.py b/evals/team-tasks/vo-x20/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x20/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x20/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x4/steps/validator/tests/finish.py b/evals/team-tasks/vo-x4/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x4/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x4/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x7/steps/validator/tests/finish.py b/evals/team-tasks/vo-x7/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x7/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x7/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x8/steps/validator/tests/finish.py b/evals/team-tasks/vo-x8/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x8/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x8/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: