From 41286d7f2f814a8c822338588d6c34b11340b391 Mon Sep 17 00:00:00 2001
From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com>
Date: Wed, 30 Sep 2026 00:28:34 +0200
Subject: [PATCH 1/3] Add prompt-hashed validation/test split to meta-eval

Judge tuning needs to select configurations on one partition of the
human-labeled arena and report on another. Battles are assigned by a
salted hash of their first user prompt, so repeated prompts never cross
partitions and changing run.seed cannot move battles between them.
---
 judgearena/benchmarks/meta_eval/runner.py   | 12 +++++++++++-
 judgearena/benchmarks/meta_eval/sampling.py | 20 ++++++++++++++++++++
 judgearena/config.py                        |  7 +++++++
 judgearena/reports.py                       |  3 ++-
 tests/test_meta_eval_foundation.py          | 14 ++++++++++++++
 5 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/judgearena/benchmarks/meta_eval/runner.py b/judgearena/benchmarks/meta_eval/runner.py
index 5c052d48..4d57d12d 100644
--- a/judgearena/benchmarks/meta_eval/runner.py
+++ b/judgearena/benchmarks/meta_eval/runner.py
@@ -10,6 +10,7 @@
 import numpy as np
 import pandas as pd
 
+from judgearena.arenas_utils import extract_turn_text
 from judgearena.artifacts import (
     prepare_run_directory,
     safe_filename,
@@ -24,6 +25,7 @@
 )
 from judgearena.benchmarks.meta_eval.sampling import (
     MetaEvalSamplingError,
+    filter_split,
     sample_battles_per_model,
     select_top_models,
 )
@@ -133,6 +135,13 @@ def run_meta_eval(
     top_models, top_pool = select_top_models(
         arena_battles, top_models=cfg.meta_eval.top_models
     )
+    if cfg.meta_eval.split != "all":
+        top_pool = filter_split(
+            top_pool,
+            top_pool["conversation_a"].map(lambda turns: extract_turn_text(turns[0])),
+            split=cfg.meta_eval.split,
+            validation_fraction=cfg.meta_eval.validation_fraction,
+        )
     sample = sample_battles_per_model(
         top_pool,
         top_models,
@@ -155,7 +164,7 @@ def run_meta_eval(
     timestamp = run_started_at.strftime("%Y%m%d_%H%M%S")
     result_name = (
         f"{safe_filename(cfg.task)}-{safe_filename(cfg.judge.model)}-"
-        f"{cfg.judge.swap_mode}-{timestamp}"
+        f"{cfg.judge.swap_mode}-{cfg.meta_eval.split}-{timestamp}"
     )
     result_dir = prepare_run_directory(cfg, Path(cfg.run.result_folder) / result_name)
     sample.loc[
@@ -198,6 +207,7 @@ def run_meta_eval(
         top_models=top_models,
         n_sampled_battles=len(sample),
         swap_mode=cfg.judge.swap_mode,
+        split=cfg.meta_eval.split,
         metrics=metric_results,
     )
     results = report.to_dict()
diff --git a/judgearena/benchmarks/meta_eval/sampling.py b/judgearena/benchmarks/meta_eval/sampling.py
index 19e36e61..efb51062 100644
--- a/judgearena/benchmarks/meta_eval/sampling.py
+++ b/judgearena/benchmarks/meta_eval/sampling.py
@@ -7,6 +7,8 @@
 
 import pandas as pd
 
+_SPLIT_SALT = "judgearena-meta-eval-split"
+
 
 class MetaEvalSamplingError(ValueError):
     """Raised when filtering or sampling yields an unusable battle subset."""
@@ -85,6 +87,24 @@ def select_top_models(
     return top, df_top
 
 
+def _validation_bucket(prompt: str) -> float:
+    digest = hashlib.sha256(f"{_SPLIT_SALT}\0{prompt}".encode()).digest()
+    return int.from_bytes(digest[:8], "big") / 2**64
+
+
+def filter_split(
+    df: pd.DataFrame, prompts: pd.Series, *, split: str, validation_fraction: float
+) -> pd.DataFrame:
+    """Keep battles whose prompt hashes into ``split``.
+
+    Keying on the prompt keeps repeated prompts on one side and makes the
+    partition independent of the sampling seed.
+    """
+    is_validation = prompts.map(_validation_bucket) < validation_fraction
+    keep = is_validation if split == "validation" else ~is_validation
+    return df.loc[keep.to_numpy()].reset_index(drop=True)
+
+
 def _stable_priority(battle_id: object, *, seed: int) -> str:
     payload = f"{seed}\0{battle_id}".encode()
     return hashlib.sha256(payload).hexdigest()
diff --git a/judgearena/config.py b/judgearena/config.py
index 74f68fbb..f0ac302e 100644
--- a/judgearena/config.py
+++ b/judgearena/config.py
@@ -403,6 +403,13 @@ class MetaEvalArgs(BaseModel):
     languages: list[str] | None = None
     """Restrict arena battles to these language codes. Defaults to all languages."""
 
+    split: Literal["all", "validation", "test"] = "all"
+    """Prompt-hashed arena partition to sample from. Judge tuning selects on
+    ``validation`` and reports on ``test``."""
+
+    validation_fraction: float = Field(default=0.5, gt=0, lt=1)
+    """Share of arena prompts assigned to the validation partition."""
+
 
 class RunArgs(BaseModel):
     """Run-level settings: seed, output location, caching, and logging."""
diff --git a/judgearena/reports.py b/judgearena/reports.py
index 783555f5..ea03982f 100644
--- a/judgearena/reports.py
+++ b/judgearena/reports.py
@@ -101,12 +101,13 @@ class MetaEvalReport(Report):
     top_models: list[str]
     n_sampled_battles: int
     swap_mode: str
+    split: str
     metrics: dict[str, dict[str, object]]
 
     def render(self) -> None:
         from judgearena.benchmarks.scoring import render_metrics
 
-        print(f"\n=== Meta-eval: {self.task} ===")
+        print(f"\n=== Meta-eval: {self.task} ({self.split} split) ===")
         print(f"Arena: {self.arena} | Judge: {self.judge_model}")
         print(
             f"Models: {len(self.top_models)} | Sampled battles: {self.n_sampled_battles} | "
diff --git a/tests/test_meta_eval_foundation.py b/tests/test_meta_eval_foundation.py
index 4233c900..798cbbac 100644
--- a/tests/test_meta_eval_foundation.py
+++ b/tests/test_meta_eval_foundation.py
@@ -11,6 +11,7 @@
 import judgearena.evaluate as evaluate_module
 from judgearena.benchmarks.meta_eval.sampling import (
     MetaEvalSamplingError,
+    filter_split,
     sample_battles_per_model,
     select_top_models,
 )
@@ -55,6 +56,19 @@ def test_sampling_rejects_insufficient_quota():
         sample_battles_per_model(battles, ["a", "b"], battles_per_model=2, seed=0)
 
 
+def test_split_partitions_battles_by_prompt():
+    battles = pd.DataFrame({"battle_id": range(200)})
+    prompts = battles["battle_id"].map(lambda i: f"prompt {i % 40}")
+    validation, test = (
+        filter_split(battles, prompts, split=split, validation_fraction=0.5)
+        for split in ("validation", "test")
+    )
+
+    assert 0 < len(validation) < len(battles)
+    assert sorted([*validation["battle_id"], *test["battle_id"]]) == list(range(200))
+    assert not set(prompts[validation["battle_id"]]) & set(prompts[test["battle_id"]])
+
+
 def test_meta_eval_protocol_requires_no_baseline_and_compatible_metrics():
     definition = get_packaged_task("meta-eval-comparia").spec.model_dump()
     protocol = definition["protocol"]

From 7654e862af95e600fbed38f8403a23f8196831cf Mon Sep 17 00:00:00 2001
From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com>
Date: Wed, 7 Oct 2026 15:11:31 +0200
Subject: [PATCH 2/3] Share arena definitions for evaluation and tuning tasks

---
 README.md                                     |  2 ++
 .../definitions/meta_eval/_comparia.yaml      | 19 ++++++++++++++++++
 .../definitions/meta_eval/_lmarena-100k.yaml  | 19 ++++++++++++++++++
 .../definitions/meta_eval/_lmarena-140k.yaml  | 19 ++++++++++++++++++
 .../meta_eval/meta-eval-comparia.yaml         | 20 +------------------
 .../meta_eval/meta-eval-lmarena-100k.yaml     | 20 +------------------
 .../meta_eval/meta-eval-lmarena-140k.yaml     | 20 +------------------
 7 files changed, 62 insertions(+), 57 deletions(-)
 create mode 100644 judgearena/tasks/definitions/meta_eval/_comparia.yaml
 create mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml
 create mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml

diff --git a/README.md b/README.md
index 50f60be8..2367ca42 100644
--- a/README.md
+++ b/README.md
@@ -337,6 +337,8 @@ judgearena \
 
 Runs save the selected sample, judge evidence, metric battles, configuration, and results under `--run.result_folder`.
 
+`--meta_eval.split validation|test` restricts sampling to one half of the arena prompts (`--meta_eval.validation_fraction`, default 0.5). The split hashes each battle's prompt, so it does not depend on `--run.seed`.
+
 ## 📈 Estimating ELO Ratings
 
 JudgeArena can estimate the ELO rating of a model by running it against opponents sampled from a human preference arena (`LMArena-100k`, `LMArena-140k`, or `ComparIA`).
diff --git a/judgearena/tasks/definitions/meta_eval/_comparia.yaml b/judgearena/tasks/definitions/meta_eval/_comparia.yaml
new file mode 100644
index 00000000..6947fd5e
--- /dev/null
+++ b/judgearena/tasks/definitions/meta_eval/_comparia.yaml
@@ -0,0 +1,19 @@
+extends: _base.yaml
+
+variants:
+  selector: language
+  values: [fr, en]
+
+dataset:
+  sources:
+    comparia:
+      type: huggingface_dataset
+      repo_id: ministere-culture/comparia-votes
+      revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b"
+      allow_patterns: [votes.parquet]
+
+protocol:
+  arena: ComparIA
+
+metadata:
+  reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes
diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml
new file mode 100644
index 00000000..dafbf9c3
--- /dev/null
+++ b/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml
@@ -0,0 +1,19 @@
+extends: _base.yaml
+
+variants:
+  selector: language
+  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
+
+dataset:
+  sources:
+    lmarena_100k:
+      type: huggingface_dataset
+      repo_id: lmarena-ai/arena-human-preference-100k
+      revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a"
+      allow_patterns: [data/*.parquet]
+
+protocol:
+  arena: LMArena-100k
+
+metadata:
+  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k
diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml
new file mode 100644
index 00000000..36ea359e
--- /dev/null
+++ b/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml
@@ -0,0 +1,19 @@
+extends: _base.yaml
+
+variants:
+  selector: language
+  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
+
+dataset:
+  sources:
+    lmarena_140k:
+      type: huggingface_dataset
+      repo_id: lmarena-ai/arena-human-preference-140k
+      revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74"
+      allow_patterns: [data/*.parquet]
+
+protocol:
+  arena: LMArena-140k
+
+metadata:
+  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k
diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml
index f9a34f46..d0d35a00 100644
--- a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml
+++ b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml
@@ -1,21 +1,3 @@
-extends: _base.yaml
+extends: _comparia.yaml
 task: meta-eval-comparia
 description: Score a judge against human votes from the ComparIA arena.
-
-variants:
-  selector: language
-  values: [fr, en]
-
-dataset:
-  sources:
-    comparia:
-      type: huggingface_dataset
-      repo_id: ministere-culture/comparia-votes
-      revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b"
-      allow_patterns: [votes.parquet]
-
-protocol:
-  arena: ComparIA
-
-metadata:
-  reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes
diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml
index 4fa21998..679d3db0 100644
--- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml
+++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml
@@ -1,21 +1,3 @@
-extends: _base.yaml
+extends: _lmarena-100k.yaml
 task: meta-eval-lmarena-100k
 description: Score a judge against human votes from LMSYS Chatbot Arena 100k.
-
-variants:
-  selector: language
-  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
-
-dataset:
-  sources:
-    lmarena_100k:
-      type: huggingface_dataset
-      repo_id: lmarena-ai/arena-human-preference-100k
-      revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a"
-      allow_patterns: [data/*.parquet]
-
-protocol:
-  arena: LMArena-100k
-
-metadata:
-  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k
diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml
index 52ba8059..2ffddfd5 100644
--- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml
+++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml
@@ -1,21 +1,3 @@
-extends: _base.yaml
+extends: _lmarena-140k.yaml
 task: meta-eval-lmarena-140k
 description: Score a judge against human votes from LMSYS Chatbot Arena 140k.
-
-variants:
-  selector: language
-  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
-
-dataset:
-  sources:
-    lmarena_140k:
-      type: huggingface_dataset
-      repo_id: lmarena-ai/arena-human-preference-140k
-      revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74"
-      allow_patterns: [data/*.parquet]
-
-protocol:
-  arena: LMArena-140k
-
-metadata:
-  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k

From 64e95b2d78b3c8612ee9d32e3d31a4c2db42ffe6 Mon Sep 17 00:00:00 2001
From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com>
Date: Thu, 8 Oct 2026 16:30:40 +0200
Subject: [PATCH 3/3] Keep arena declarations unchanged in the split PR

---
 .../definitions/meta_eval/_comparia.yaml      | 19 ------------------
 .../definitions/meta_eval/_lmarena-100k.yaml  | 19 ------------------
 .../definitions/meta_eval/_lmarena-140k.yaml  | 19 ------------------
 .../meta_eval/meta-eval-comparia.yaml         | 20 ++++++++++++++++++-
 .../meta_eval/meta-eval-lmarena-100k.yaml     | 20 ++++++++++++++++++-
 .../meta_eval/meta-eval-lmarena-140k.yaml     | 20 ++++++++++++++++++-
 6 files changed, 57 insertions(+), 60 deletions(-)
 delete mode 100644 judgearena/tasks/definitions/meta_eval/_comparia.yaml
 delete mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml
 delete mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml

diff --git a/judgearena/tasks/definitions/meta_eval/_comparia.yaml b/judgearena/tasks/definitions/meta_eval/_comparia.yaml
deleted file mode 100644
index 6947fd5e..00000000
--- a/judgearena/tasks/definitions/meta_eval/_comparia.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-extends: _base.yaml
-
-variants:
-  selector: language
-  values: [fr, en]
-
-dataset:
-  sources:
-    comparia:
-      type: huggingface_dataset
-      repo_id: ministere-culture/comparia-votes
-      revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b"
-      allow_patterns: [votes.parquet]
-
-protocol:
-  arena: ComparIA
-
-metadata:
-  reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes
diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml
deleted file mode 100644
index dafbf9c3..00000000
--- a/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-extends: _base.yaml
-
-variants:
-  selector: language
-  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
-
-dataset:
-  sources:
-    lmarena_100k:
-      type: huggingface_dataset
-      repo_id: lmarena-ai/arena-human-preference-100k
-      revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a"
-      allow_patterns: [data/*.parquet]
-
-protocol:
-  arena: LMArena-100k
-
-metadata:
-  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k
diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml
deleted file mode 100644
index 36ea359e..00000000
--- a/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-extends: _base.yaml
-
-variants:
-  selector: language
-  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
-
-dataset:
-  sources:
-    lmarena_140k:
-      type: huggingface_dataset
-      repo_id: lmarena-ai/arena-human-preference-140k
-      revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74"
-      allow_patterns: [data/*.parquet]
-
-protocol:
-  arena: LMArena-140k
-
-metadata:
-  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k
diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml
index d0d35a00..f9a34f46 100644
--- a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml
+++ b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml
@@ -1,3 +1,21 @@
-extends: _comparia.yaml
+extends: _base.yaml
 task: meta-eval-comparia
 description: Score a judge against human votes from the ComparIA arena.
+
+variants:
+  selector: language
+  values: [fr, en]
+
+dataset:
+  sources:
+    comparia:
+      type: huggingface_dataset
+      repo_id: ministere-culture/comparia-votes
+      revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b"
+      allow_patterns: [votes.parquet]
+
+protocol:
+  arena: ComparIA
+
+metadata:
+  reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes
diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml
index 679d3db0..4fa21998 100644
--- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml
+++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml
@@ -1,3 +1,21 @@
-extends: _lmarena-100k.yaml
+extends: _base.yaml
 task: meta-eval-lmarena-100k
 description: Score a judge against human votes from LMSYS Chatbot Arena 100k.
+
+variants:
+  selector: language
+  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
+
+dataset:
+  sources:
+    lmarena_100k:
+      type: huggingface_dataset
+      repo_id: lmarena-ai/arena-human-preference-100k
+      revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a"
+      allow_patterns: [data/*.parquet]
+
+protocol:
+  arena: LMArena-100k
+
+metadata:
+  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k
diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml
index 2ffddfd5..52ba8059 100644
--- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml
+++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml
@@ -1,3 +1,21 @@
-extends: _lmarena-140k.yaml
+extends: _base.yaml
 task: meta-eval-lmarena-140k
 description: Score a judge against human votes from LMSYS Chatbot Arena 140k.
+
+variants:
+  selector: language
+  values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk]
+
+dataset:
+  sources:
+    lmarena_140k:
+      type: huggingface_dataset
+      repo_id: lmarena-ai/arena-human-preference-140k
+      revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74"
+      allow_patterns: [data/*.parquet]
+
+protocol:
+  arena: LMArena-140k
+
+metadata:
+  reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k