diff --git a/evals/team-tasks/vo-t01/steps/validator/tests/finish.py b/evals/team-tasks/vo-t01/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t01/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t01/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t02/steps/validator/tests/finish.py b/evals/team-tasks/vo-t02/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t02/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t02/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t03/steps/validator/tests/finish.py b/evals/team-tasks/vo-t03/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t03/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t03/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t04/steps/validator/tests/finish.py b/evals/team-tasks/vo-t04/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t04/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t04/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t05/steps/validator/tests/finish.py b/evals/team-tasks/vo-t05/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t05/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t05/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-t06/steps/validator/tests/finish.py b/evals/team-tasks/vo-t06/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-t06/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-t06/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x1/steps/validator/tests/finish.py b/evals/team-tasks/vo-x1/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x1/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x1/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x17/steps/validator/tests/finish.py b/evals/team-tasks/vo-x17/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x17/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x17/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x20/steps/validator/tests/finish.py b/evals/team-tasks/vo-x20/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x20/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x20/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x4/steps/validator/tests/finish.py b/evals/team-tasks/vo-x4/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x4/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x4/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x7/steps/validator/tests/finish.py b/evals/team-tasks/vo-x7/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x7/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x7/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: diff --git a/evals/team-tasks/vo-x8/steps/validator/tests/finish.py b/evals/team-tasks/vo-x8/steps/validator/tests/finish.py index c2416d1..ee92b92 100644 --- a/evals/team-tasks/vo-x8/steps/validator/tests/finish.py +++ b/evals/team-tasks/vo-x8/steps/validator/tests/finish.py @@ -42,7 +42,17 @@ def load_findings(path: Path) -> tuple[list | None, str]: return data, "" -def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: +def give_api_judges_the_file(rubric: Path, findings_file: Path) -> None: + """List the findings file in each judge.toml: an API judge cannot open files, and Reward Kit + sends a listed one as user content; agent judges ignore it and open the file.""" + for toml in rubric.rglob("*.toml"): + text = toml.read_text(encoding="utf-8") + if "[judge]\n" in text: + line = f"files = [{json.dumps(str(findings_file))}]\n" + toml.write_text(text.replace("[judge]\n", "[judge]\n" + line, 1), encoding="utf-8") + + +def rubric_with_findings(rubric: Path, findings: list, work: Path, findings_file: Path) -> Path: """A copy of the rubric whose prompts hold the findings, so no judge can grade without them. An agent judge is only pointed at its working directory and may answer without opening @@ -56,6 +66,7 @@ def rubric_with_findings(rubric: Path, findings: list, work: Path) -> Path: text = text.replace(raw, escaped) if len(text.encode("utf-8")) > MAX_PASTE: text = TOO_LONG + give_api_judges_the_file(copy, findings_file) prompts = [p for p in copy.rglob("*.md") if SLOT in p.read_text(encoding="utf-8")] if not prompts: raise ValueError(f"no rubric prompt under {rubric} has a {SLOT} slot") @@ -111,7 +122,8 @@ def finish( (workspace / FINDINGS).write_text(json.dumps(findings, indent=2) + "\n") output = work / "reward.json" try: - judge(rubric_with_findings(tests / "rubric", findings, work), workspace, output) + rubric = rubric_with_findings(tests / "rubric", findings, work, workspace / FINDINGS) + judge(rubric, workspace, output) scores = json.loads(output.read_text()) details = json.loads(output.with_name("reward-details.json").read_text()) except (subprocess.CalledProcessError, OSError, ValueError) as exc: