WIP: Judge tuning (1/6): Add held-out arena partitions

README.md

@@ -337,6 +337,8 @@ judgearena \
 
 Runs save the selected sample, judge evidence, metric battles, configuration, and results under `--run.result_folder`.
 
+`--meta_eval.split validation|test` restricts sampling to one half of the arena prompts (`--meta_eval.validation_fraction`, default 0.5). The split hashes each battle's prompt, so it does not depend on `--run.seed`.
+
 ## 📈 Estimating ELO Ratings
 
 JudgeArena can estimate the ELO rating of a model by running it against opponents sampled from a human preference arena (`LMArena-100k`, `LMArena-140k`, or `ComparIA`).

judgearena/benchmarks/meta_eval/runner.py

@@ -10,6 +10,7 @@
 import numpy as np
 import pandas as pd
 
+from judgearena.arenas_utils import extract_turn_text
 from judgearena.artifacts import (
     prepare_run_directory,
     safe_filename,
@@ -24,6 +25,7 @@
 )
 from judgearena.benchmarks.meta_eval.sampling import (
     MetaEvalSamplingError,
+    filter_split,
     sample_battles_per_model,
     select_top_models,
 )
@@ -133,6 +135,13 @@ def run_meta_eval(
     top_models, top_pool = select_top_models(
         arena_battles, top_models=cfg.meta_eval.top_models
     )
+    if cfg.meta_eval.split != "all":
+        top_pool = filter_split(
+            top_pool,
+            top_pool["conversation_a"].map(lambda turns: extract_turn_text(turns[0])),
+            split=cfg.meta_eval.split,
+            validation_fraction=cfg.meta_eval.validation_fraction,
+        )
     sample = sample_battles_per_model(
         top_pool,
         top_models,
@@ -155,7 +164,7 @@ def run_meta_eval(
     timestamp = run_started_at.strftime("%Y%m%d_%H%M%S")
     result_name = (
         f"{safe_filename(cfg.task)}-{safe_filename(cfg.judge.model)}-"
-        f"{cfg.judge.swap_mode}-{timestamp}"
+        f"{cfg.judge.swap_mode}-{cfg.meta_eval.split}-{timestamp}"
     )
     result_dir = prepare_run_directory(cfg, Path(cfg.run.result_folder) / result_name)
     sample.loc[
@@ -198,6 +207,7 @@ def run_meta_eval(
         top_models=top_models,
         n_sampled_battles=len(sample),
         swap_mode=cfg.judge.swap_mode,
+        split=cfg.meta_eval.split,
         metrics=metric_results,
     )
     results = report.to_dict()

judgearena/benchmarks/meta_eval/sampling.py

@@ -7,6 +7,8 @@
 
 import pandas as pd
 
+_SPLIT_SALT = "judgearena-meta-eval-split"
+
 
 class MetaEvalSamplingError(ValueError):
     """Raised when filtering or sampling yields an unusable battle subset."""
@@ -85,6 +87,24 @@ def select_top_models(
     return top, df_top
 
 
+def _validation_bucket(prompt: str) -> float:
+    digest = hashlib.sha256(f"{_SPLIT_SALT}\0{prompt}".encode()).digest()
+    return int.from_bytes(digest[:8], "big") / 2**64
+
+
+def filter_split(
+    df: pd.DataFrame, prompts: pd.Series, *, split: str, validation_fraction: float
+) -> pd.DataFrame:
+    """Keep battles whose prompt hashes into ``split``.
+
+    Keying on the prompt keeps repeated prompts on one side and makes the
+    partition independent of the sampling seed.
+    """
+    is_validation = prompts.map(_validation_bucket) < validation_fraction
+    keep = is_validation if split == "validation" else ~is_validation
+    return df.loc[keep.to_numpy()].reset_index(drop=True)
+
+
 def _stable_priority(battle_id: object, *, seed: int) -> str:
     payload = f"{seed}\0{battle_id}".encode()
     return hashlib.sha256(payload).hexdigest()

judgearena/config.py

@@ -403,6 +403,13 @@ class MetaEvalArgs(BaseModel):
     languages: list[str] | None = None
     """Restrict arena battles to these language codes. Defaults to all languages."""
 
+    split: Literal["all", "validation", "test"] = "all"
+    """Prompt-hashed arena partition to sample from. Judge tuning selects on
+    ``validation`` and reports on ``test``."""
+
+    validation_fraction: float = Field(default=0.5, gt=0, lt=1)
+    """Share of arena prompts assigned to the validation partition."""
+
 
 class RunArgs(BaseModel):
     """Run-level settings: seed, output location, caching, and logging."""

judgearena/reports.py

@@ -101,12 +101,13 @@ class MetaEvalReport(Report):
     top_models: list[str]
     n_sampled_battles: int
     swap_mode: str
+    split: str
     metrics: dict[str, dict[str, object]]
 
     def render(self) -> None:
         from judgearena.benchmarks.scoring import render_metrics
 
-        print(f"\n=== Meta-eval: {self.task} ===")
+        print(f"\n=== Meta-eval: {self.task} ({self.split} split) ===")
         print(f"Arena: {self.arena} | Judge: {self.judge_model}")
         print(
             f"Models: {len(self.top_models)} | Sampled battles: {self.n_sampled_battles} | "

tests/test_meta_eval_foundation.py

@@ -11,6 +11,7 @@
 import judgearena.evaluate as evaluate_module
 from judgearena.benchmarks.meta_eval.sampling import (
     MetaEvalSamplingError,
+    filter_split,
     sample_battles_per_model,
     select_top_models,
 )
@@ -55,6 +56,19 @@ def test_sampling_rejects_insufficient_quota():
         sample_battles_per_model(battles, ["a", "b"], battles_per_model=2, seed=0)
 
 
+def test_split_partitions_battles_by_prompt():
+    battles = pd.DataFrame({"battle_id": range(200)})
+    prompts = battles["battle_id"].map(lambda i: f"prompt {i % 40}")
+    validation, test = (
+        filter_split(battles, prompts, split=split, validation_fraction=0.5)
+        for split in ("validation", "test")
+    )
+
+    assert 0 < len(validation) < len(battles)
+    assert sorted([*validation["battle_id"], *test["battle_id"]]) == list(range(200))
+    assert not set(prompts[validation["battle_id"]]) & set(prompts[test["battle_id"]])
+
+
 def test_meta_eval_protocol_requires_no_baseline_and_compatible_metrics():
     definition = get_packaged_task("meta-eval-comparia").spec.model_dump()
     protocol = definition["protocol"]