From 41286d7f2f814a8c822338588d6c34b11340b391 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:28:34 +0200 Subject: [PATCH 01/13] Add prompt-hashed validation/test split to meta-eval Judge tuning needs to select configurations on one partition of the human-labeled arena and report on another. Battles are assigned by a salted hash of their first user prompt, so repeated prompts never cross partitions and changing run.seed cannot move battles between them. --- judgearena/benchmarks/meta_eval/runner.py | 12 +++++++++++- judgearena/benchmarks/meta_eval/sampling.py | 20 ++++++++++++++++++++ judgearena/config.py | 7 +++++++ judgearena/reports.py | 3 ++- tests/test_meta_eval_foundation.py | 14 ++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/judgearena/benchmarks/meta_eval/runner.py b/judgearena/benchmarks/meta_eval/runner.py index 5c052d48..4d57d12d 100644 --- a/judgearena/benchmarks/meta_eval/runner.py +++ b/judgearena/benchmarks/meta_eval/runner.py @@ -10,6 +10,7 @@ import numpy as np import pandas as pd +from judgearena.arenas_utils import extract_turn_text from judgearena.artifacts import ( prepare_run_directory, safe_filename, @@ -24,6 +25,7 @@ ) from judgearena.benchmarks.meta_eval.sampling import ( MetaEvalSamplingError, + filter_split, sample_battles_per_model, select_top_models, ) @@ -133,6 +135,13 @@ def run_meta_eval( top_models, top_pool = select_top_models( arena_battles, top_models=cfg.meta_eval.top_models ) + if cfg.meta_eval.split != "all": + top_pool = filter_split( + top_pool, + top_pool["conversation_a"].map(lambda turns: extract_turn_text(turns[0])), + split=cfg.meta_eval.split, + validation_fraction=cfg.meta_eval.validation_fraction, + ) sample = sample_battles_per_model( top_pool, top_models, @@ -155,7 +164,7 @@ def run_meta_eval( timestamp = run_started_at.strftime("%Y%m%d_%H%M%S") result_name = ( f"{safe_filename(cfg.task)}-{safe_filename(cfg.judge.model)}-" - f"{cfg.judge.swap_mode}-{timestamp}" + f"{cfg.judge.swap_mode}-{cfg.meta_eval.split}-{timestamp}" ) result_dir = prepare_run_directory(cfg, Path(cfg.run.result_folder) / result_name) sample.loc[ @@ -198,6 +207,7 @@ def run_meta_eval( top_models=top_models, n_sampled_battles=len(sample), swap_mode=cfg.judge.swap_mode, + split=cfg.meta_eval.split, metrics=metric_results, ) results = report.to_dict() diff --git a/judgearena/benchmarks/meta_eval/sampling.py b/judgearena/benchmarks/meta_eval/sampling.py index 19e36e61..efb51062 100644 --- a/judgearena/benchmarks/meta_eval/sampling.py +++ b/judgearena/benchmarks/meta_eval/sampling.py @@ -7,6 +7,8 @@ import pandas as pd +_SPLIT_SALT = "judgearena-meta-eval-split" + class MetaEvalSamplingError(ValueError): """Raised when filtering or sampling yields an unusable battle subset.""" @@ -85,6 +87,24 @@ def select_top_models( return top, df_top +def _validation_bucket(prompt: str) -> float: + digest = hashlib.sha256(f"{_SPLIT_SALT}\0{prompt}".encode()).digest() + return int.from_bytes(digest[:8], "big") / 2**64 + + +def filter_split( + df: pd.DataFrame, prompts: pd.Series, *, split: str, validation_fraction: float +) -> pd.DataFrame: + """Keep battles whose prompt hashes into ``split``. + + Keying on the prompt keeps repeated prompts on one side and makes the + partition independent of the sampling seed. + """ + is_validation = prompts.map(_validation_bucket) < validation_fraction + keep = is_validation if split == "validation" else ~is_validation + return df.loc[keep.to_numpy()].reset_index(drop=True) + + def _stable_priority(battle_id: object, *, seed: int) -> str: payload = f"{seed}\0{battle_id}".encode() return hashlib.sha256(payload).hexdigest() diff --git a/judgearena/config.py b/judgearena/config.py index 74f68fbb..f0ac302e 100644 --- a/judgearena/config.py +++ b/judgearena/config.py @@ -403,6 +403,13 @@ class MetaEvalArgs(BaseModel): languages: list[str] | None = None """Restrict arena battles to these language codes. Defaults to all languages.""" + split: Literal["all", "validation", "test"] = "all" + """Prompt-hashed arena partition to sample from. Judge tuning selects on + ``validation`` and reports on ``test``.""" + + validation_fraction: float = Field(default=0.5, gt=0, lt=1) + """Share of arena prompts assigned to the validation partition.""" + class RunArgs(BaseModel): """Run-level settings: seed, output location, caching, and logging.""" diff --git a/judgearena/reports.py b/judgearena/reports.py index 783555f5..ea03982f 100644 --- a/judgearena/reports.py +++ b/judgearena/reports.py @@ -101,12 +101,13 @@ class MetaEvalReport(Report): top_models: list[str] n_sampled_battles: int swap_mode: str + split: str metrics: dict[str, dict[str, object]] def render(self) -> None: from judgearena.benchmarks.scoring import render_metrics - print(f"\n=== Meta-eval: {self.task} ===") + print(f"\n=== Meta-eval: {self.task} ({self.split} split) ===") print(f"Arena: {self.arena} | Judge: {self.judge_model}") print( f"Models: {len(self.top_models)} | Sampled battles: {self.n_sampled_battles} | " diff --git a/tests/test_meta_eval_foundation.py b/tests/test_meta_eval_foundation.py index 4233c900..798cbbac 100644 --- a/tests/test_meta_eval_foundation.py +++ b/tests/test_meta_eval_foundation.py @@ -11,6 +11,7 @@ import judgearena.evaluate as evaluate_module from judgearena.benchmarks.meta_eval.sampling import ( MetaEvalSamplingError, + filter_split, sample_battles_per_model, select_top_models, ) @@ -55,6 +56,19 @@ def test_sampling_rejects_insufficient_quota(): sample_battles_per_model(battles, ["a", "b"], battles_per_model=2, seed=0) +def test_split_partitions_battles_by_prompt(): + battles = pd.DataFrame({"battle_id": range(200)}) + prompts = battles["battle_id"].map(lambda i: f"prompt {i % 40}") + validation, test = ( + filter_split(battles, prompts, split=split, validation_fraction=0.5) + for split in ("validation", "test") + ) + + assert 0 < len(validation) < len(battles) + assert sorted([*validation["battle_id"], *test["battle_id"]]) == list(range(200)) + assert not set(prompts[validation["battle_id"]]) & set(prompts[test["battle_id"]]) + + def test_meta_eval_protocol_requires_no_baseline_and_compatible_metrics(): definition = get_packaged_task("meta-eval-comparia").spec.model_dump() protocol = definition["protocol"] From 7654e862af95e600fbed38f8403a23f8196831cf Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:11:31 +0200 Subject: [PATCH 02/13] Share arena definitions for evaluation and tuning tasks --- README.md | 2 ++ .../definitions/meta_eval/_comparia.yaml | 19 ++++++++++++++++++ .../definitions/meta_eval/_lmarena-100k.yaml | 19 ++++++++++++++++++ .../definitions/meta_eval/_lmarena-140k.yaml | 19 ++++++++++++++++++ .../meta_eval/meta-eval-comparia.yaml | 20 +------------------ .../meta_eval/meta-eval-lmarena-100k.yaml | 20 +------------------ .../meta_eval/meta-eval-lmarena-140k.yaml | 20 +------------------ 7 files changed, 62 insertions(+), 57 deletions(-) create mode 100644 judgearena/tasks/definitions/meta_eval/_comparia.yaml create mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml create mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml diff --git a/README.md b/README.md index 50f60be8..2367ca42 100644 --- a/README.md +++ b/README.md @@ -337,6 +337,8 @@ judgearena \ Runs save the selected sample, judge evidence, metric battles, configuration, and results under `--run.result_folder`. +`--meta_eval.split validation|test` restricts sampling to one half of the arena prompts (`--meta_eval.validation_fraction`, default 0.5). The split hashes each battle's prompt, so it does not depend on `--run.seed`. + ## 📈 Estimating ELO Ratings JudgeArena can estimate the ELO rating of a model by running it against opponents sampled from a human preference arena (`LMArena-100k`, `LMArena-140k`, or `ComparIA`). diff --git a/judgearena/tasks/definitions/meta_eval/_comparia.yaml b/judgearena/tasks/definitions/meta_eval/_comparia.yaml new file mode 100644 index 00000000..6947fd5e --- /dev/null +++ b/judgearena/tasks/definitions/meta_eval/_comparia.yaml @@ -0,0 +1,19 @@ +extends: _base.yaml + +variants: + selector: language + values: [fr, en] + +dataset: + sources: + comparia: + type: huggingface_dataset + repo_id: ministere-culture/comparia-votes + revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b" + allow_patterns: [votes.parquet] + +protocol: + arena: ComparIA + +metadata: + reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml new file mode 100644 index 00000000..dafbf9c3 --- /dev/null +++ b/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml @@ -0,0 +1,19 @@ +extends: _base.yaml + +variants: + selector: language + values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] + +dataset: + sources: + lmarena_100k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-100k + revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" + allow_patterns: [data/*.parquet] + +protocol: + arena: LMArena-100k + +metadata: + reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml new file mode 100644 index 00000000..36ea359e --- /dev/null +++ b/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml @@ -0,0 +1,19 @@ +extends: _base.yaml + +variants: + selector: language + values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] + +dataset: + sources: + lmarena_140k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-140k + revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74" + allow_patterns: [data/*.parquet] + +protocol: + arena: LMArena-140k + +metadata: + reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml index f9a34f46..d0d35a00 100644 --- a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml +++ b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml @@ -1,21 +1,3 @@ -extends: _base.yaml +extends: _comparia.yaml task: meta-eval-comparia description: Score a judge against human votes from the ComparIA arena. - -variants: - selector: language - values: [fr, en] - -dataset: - sources: - comparia: - type: huggingface_dataset - repo_id: ministere-culture/comparia-votes - revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b" - allow_patterns: [votes.parquet] - -protocol: - arena: ComparIA - -metadata: - reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml index 4fa21998..679d3db0 100644 --- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml +++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml @@ -1,21 +1,3 @@ -extends: _base.yaml +extends: _lmarena-100k.yaml task: meta-eval-lmarena-100k description: Score a judge against human votes from LMSYS Chatbot Arena 100k. - -variants: - selector: language - values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] - -dataset: - sources: - lmarena_100k: - type: huggingface_dataset - repo_id: lmarena-ai/arena-human-preference-100k - revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" - allow_patterns: [data/*.parquet] - -protocol: - arena: LMArena-100k - -metadata: - reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml index 52ba8059..2ffddfd5 100644 --- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml +++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml @@ -1,21 +1,3 @@ -extends: _base.yaml +extends: _lmarena-140k.yaml task: meta-eval-lmarena-140k description: Score a judge against human votes from LMSYS Chatbot Arena 140k. - -variants: - selector: language - values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] - -dataset: - sources: - lmarena_140k: - type: huggingface_dataset - repo_id: lmarena-ai/arena-human-preference-140k - revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74" - allow_patterns: [data/*.parquet] - -protocol: - arena: LMArena-140k - -metadata: - reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k From a778494bf21f84f784faddd5c35dc7f2e293f798 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:12:10 +0200 Subject: [PATCH 03/13] Add native NePS configuration and persisted tuning sessions --- judgearena/config.py | 83 ++++++++++++++++++++++++- judgearena/tasks/schema/__init__.py | 3 +- judgearena/tasks/schema/meta_eval.py | 8 +++ judgearena/tasks/schema/task.py | 5 +- judgearena/tuning/__init__.py | 1 + judgearena/tuning/search_space.py | 76 +++++++++++++++++++++++ judgearena/tuning/session.py | 93 ++++++++++++++++++++++++++++ pyproject.toml | 3 + tests/test_tuning_config.py | 91 +++++++++++++++++++++++++++ 9 files changed, 359 insertions(+), 4 deletions(-) create mode 100644 judgearena/tuning/__init__.py create mode 100644 judgearena/tuning/search_space.py create mode 100644 judgearena/tuning/session.py create mode 100644 tests/test_tuning_config.py diff --git a/judgearena/config.py b/judgearena/config.py index f0ac302e..0719332d 100644 --- a/judgearena/config.py +++ b/judgearena/config.py @@ -5,7 +5,7 @@ import argparse import os from pathlib import Path -from typing import Literal +from typing import Any, Literal import yaml from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -26,6 +26,7 @@ EloScoringSpec, MetaEvalProtocol, MTBenchProtocol, + TuneJudgeProtocol, ) # Set by build_run_config() for the duration of RunConfig() construction. @@ -411,6 +412,77 @@ class MetaEvalArgs(BaseModel): """Share of arena prompts assigned to the validation partition.""" +class TuneJudgeArgs(BaseModel): + """NePS search settings and judge-evaluation objectives.""" + + model_config = ConfigDict(use_attribute_docstrings=True, extra="forbid") + + neps: dict[str, Any] + """NePS constructor specs in pipeline_space and native neps.run arguments.""" + + objectives: list[Literal["agreement", "cost_per_1k_battles"]] = Field( + default=["agreement", "cost_per_1k_battles"], min_length=1 + ) + """Ordered objectives; agreement is maximized and expected cost minimized.""" + + price_per_million_tokens: dict[str, float] + """USD per million judge tokens, including cached judgements.""" + + test_battles_per_model: int | None = Field(default=None, gt=0) + """Test budget; defaults to the highest fidelity.""" + + run_dir: Path | None = None + """Explicit directory to resume or share; otherwise create a fresh run.""" + + search_only: bool = False + """Join an initialized run as a validation-only worker.""" + + @model_validator(mode="after") + def _validate_runtime(self) -> TuneJudgeArgs: + reserved = { + "evaluate_pipeline", + "root_directory", + "overwrite_root_directory", + "continue_until_max_evaluation_completed", + "total_cost_to_spend", + "worker_cost_to_spend", + } + if forbidden := reserved.intersection(self.neps): + raise ValueError( + f"Unsupported or managed NePS options: {sorted(forbidden)}" + ) + if self.search_only and self.run_dir is None: + raise ValueError("search_only requires an explicit tune_judge.run_dir") + if not self.search_only: + if not any( + self.neps.get(k) + for k in ("total_evaluations_to_spend", "total_fidelities_to_spend") + ): + raise ValueError("The primary worker requires a finite global budget") + if any( + self.neps.get(k) is not None + for k in ("worker_evaluations_to_spend", "worker_fidelities_to_spend") + ): + raise ValueError("Worker-local budgets require search_only") + optimizer = self.neps.get("optimizer", "auto") + name = ( + optimizer.get("name") + if isinstance(optimizer, dict) + else optimizer[0] + if isinstance(optimizer, (list, tuple)) + else optimizer + ) + if name in {"neps_priorband", "neps_hyperband"} and len(self.objectives) != 1: + raise ValueError(f"{name} requires a single objective") + if name in {"mo_hyperband", "primo"} and len(self.objectives) != 2: + raise ValueError(f"{name} requires agreement and cost objectives") + if len(set(self.objectives)) != len(self.objectives): + raise ValueError("objectives must not contain duplicates") + if any(price < 0 for price in self.price_per_million_tokens.values()): + raise ValueError("Judge token prices must be nonnegative") + return self + + class RunArgs(BaseModel): """Run-level settings: seed, output location, caching, and logging.""" @@ -467,6 +539,9 @@ class RunConfig(BaseSettings): meta_eval: MetaEvalArgs | None = None """Runtime settings used only by tasks with a meta-evaluation protocol.""" + tune_judge: TuneJudgeArgs | None = None + """Tune the judge on a meta-evaluation task instead of scoring it once.""" + run: RunArgs = Field(default_factory=RunArgs) """Run-level settings (seed, output, caching, logging).""" @@ -556,6 +631,12 @@ def _validate(self) -> RunConfig: raise ValueError( "meta_eval config is only valid for meta-evaluation tasks." ) + is_tune_judge = isinstance(protocol, TuneJudgeProtocol) + if (self.tune_judge is not None) != is_tune_judge: + raise ValueError( + "tune_judge config is required by, and only valid for, " + "tune-judge tasks." + ) if is_elo: if self.elo is None: self.elo = EloArgs() diff --git a/judgearena/tasks/schema/__init__.py b/judgearena/tasks/schema/__init__.py index 2c82226f..90039f45 100644 --- a/judgearena/tasks/schema/__init__.py +++ b/judgearena/tasks/schema/__init__.py @@ -15,7 +15,7 @@ ) from judgearena.tasks.schema.dataset import DatasetFields, DatasetSpec from judgearena.tasks.schema.elo import EloProtocol, EloScoringSpec -from judgearena.tasks.schema.meta_eval import MetaEvalProtocol +from judgearena.tasks.schema.meta_eval import MetaEvalProtocol, TuneJudgeProtocol from judgearena.tasks.schema.metrics import MetricSpec, ScoringSpec from judgearena.tasks.schema.mt_bench import ( MTBenchJudgeSpec, @@ -88,4 +88,5 @@ "TaskProvenance", "TaskSelection", "TaskSpec", + "TuneJudgeProtocol", ] diff --git a/judgearena/tasks/schema/meta_eval.py b/judgearena/tasks/schema/meta_eval.py index 3c01d120..6f0b52d6 100644 --- a/judgearena/tasks/schema/meta_eval.py +++ b/judgearena/tasks/schema/meta_eval.py @@ -39,3 +39,11 @@ def _validate_scoring(self) -> MetaEvalProtocol: f"meta-evaluation metric {request.metric} does not support breakdown_by" ) return self + + +class TuneJudgeProtocol(MetaEvalProtocol): + """Search judge settings by running trials of a meta-evaluation task.""" + + runner: Literal["tune_judge"] + meta_eval_task: str = Field(min_length=1) + """Task each trial runs as, so trials share its judgement cache.""" diff --git a/judgearena/tasks/schema/task.py b/judgearena/tasks/schema/task.py index 6f5ab80e..a9f6e61e 100644 --- a/judgearena/tasks/schema/task.py +++ b/judgearena/tasks/schema/task.py @@ -13,7 +13,7 @@ ) from judgearena.tasks.schema.dataset import DatasetSpec from judgearena.tasks.schema.elo import EloProtocol -from judgearena.tasks.schema.meta_eval import MetaEvalProtocol +from judgearena.tasks.schema.meta_eval import MetaEvalProtocol, TuneJudgeProtocol from judgearena.tasks.schema.mt_bench import MTBenchProtocol from judgearena.tasks.schema.mt_bench_101 import MTBench101Protocol from judgearena.tasks.schema.pairwise import PairwiseProtocol @@ -23,7 +23,8 @@ | MTBenchProtocol | MTBench101Protocol | EloProtocol - | MetaEvalProtocol, + | MetaEvalProtocol + | TuneJudgeProtocol, Field(discriminator="runner"), ] diff --git a/judgearena/tuning/__init__.py b/judgearena/tuning/__init__.py new file mode 100644 index 00000000..a2fa5b7b --- /dev/null +++ b/judgearena/tuning/__init__.py @@ -0,0 +1 @@ +"""Judge configuration tuning against human-labeled arena battles.""" diff --git a/judgearena/tuning/search_space.py b/judgearena/tuning/search_space.py new file mode 100644 index 00000000..3ce6720b --- /dev/null +++ b/judgearena/tuning/search_space.py @@ -0,0 +1,76 @@ +"""Construct NePS domains using its public constructor arguments.""" + +from __future__ import annotations + +import copy +from collections.abc import Iterator, Mapping +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import neps + +FIDELITY = "meta_eval.battles_per_model" +DOMAIN_TYPES = frozenset({"Categorical", "Float", "Integer", "IntegerFidelity"}) + + +def build_neps_space(specs: Mapping[str, dict]) -> neps.PipelineSpace: + """Translate constructor tags to domains without changing their arguments.""" + import neps + + parameters = {} + for name, spec in specs.items(): + kind = spec["type"] + if kind not in DOMAIN_TYPES: + raise ValueError(f"Unsupported NePS domain: {kind}") + if name == FIDELITY: + if kind != "IntegerFidelity": + raise ValueError(f"{FIDELITY} must use IntegerFidelity") + elif not ( + name.startswith("judge.") or name == "generation.truncate_judge_input_chars" + ): + raise ValueError(f"Search space must not override tuner-owned keys: {name}") + elif kind == "IntegerFidelity": + raise ValueError(f"Only {FIDELITY} may be a fidelity") + kwargs = {key: value for key, value in spec.items() if key != "type"} + if kind == "Categorical": + kwargs["choices"] = tuple(kwargs["choices"]) + parameters[name] = getattr(neps, kind)(**kwargs) + if FIDELITY not in parameters: + raise ValueError(f"Search space requires {FIDELITY}") + return type("JudgeSpace", (neps.PipelineSpace,), parameters)() + + +def axis_overrides(specs: Mapping[str, dict]) -> Iterator[dict[str, object]]: + """Yield candidate values for preflight RunConfig validation.""" + for name, spec in specs.items(): + if name == FIDELITY: + continue + values = ( + spec["choices"] + if spec["type"] == "Categorical" + else [spec["lower"], spec["upper"]] + ) + for value in values: + yield {name: value} + + +def decode_config(config: Mapping[str, Any]) -> tuple[dict[str, object], int]: + """Split the sampled judge settings from the evaluation fidelity.""" + overrides = dict(config) + return overrides, int(overrides.pop(FIDELITY)) + + +def apply_overrides( + base: Mapping[str, object], overrides: Mapping[str, object] +) -> dict[str, object]: + """Return a copy of base with dotted-path overrides assigned.""" + values = copy.deepcopy(dict(base)) + for path, value in overrides.items(): + *parents, leaf = path.split(".") + node = values + for key in parents: + if node.get(key) is None: + node[key] = {} + node = node[key] + node[leaf] = value + return values diff --git a/judgearena/tuning/session.py b/judgearena/tuning/session.py new file mode 100644 index 00000000..931d388f --- /dev/null +++ b/judgearena/tuning/session.py @@ -0,0 +1,93 @@ +"""Explicit tuning sessions and collection of shared NePS state.""" + +from __future__ import annotations + +import hashlib +import json +import time +from datetime import UTC, datetime +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +import pandas as pd + +from judgearena.artifacts import prepare_run_directory, safe_filename +from judgearena.config import RunConfig + + +def _config_hash(cfg: RunConfig) -> str: + payload = cfg.model_dump(mode="json") + payload["run"] = {"seed": cfg.run.seed} + tuning = payload["tune_judge"] + tuning.pop("run_dir") + tuning.pop("search_only") + tuning["neps"] = { + key: value + for key, value in tuning["neps"].items() + if not key.endswith("_to_spend") and key != "worker_id" + } + if cfg.judge.prompt is not None: + payload["prompt_contents"] = [ + path.read_text() + for path in (cfg.judge.prompt.system_file, cfg.judge.prompt.user_file) + ] + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +def prepare_session(cfg: RunConfig) -> Path: + """Create a new run or verify an explicitly selected existing run.""" + tuning = cfg.tune_judge + tune_dir = tuning.run_dir or ( + Path(cfg.run.result_folder) + / f"{safe_filename(cfg.task)}-{datetime.now(UTC):%Y%m%d_%H%M%S_%f}" + ) + tune_dir = tune_dir.resolve() + metadata_path = tune_dir / "tuning-metadata.json" + fingerprint = _config_hash(cfg) + if metadata_path.exists(): + metadata = json.loads(metadata_path.read_text()) + if metadata["config_hash"] != fingerprint: + raise ValueError(f"Incompatible configuration for tuning run {tune_dir}") + if not tuning.search_only: + prepare_run_directory(cfg, tune_dir) + elif tuning.search_only: + raise ValueError("Start the primary process before joining its run_dir") + else: + prepare_run_directory(cfg, tune_dir) + versions = {"neural-pipeline-search": version("neural-pipeline-search")} + try: + versions["vllm"] = version("vllm") + except PackageNotFoundError: + pass + metadata = {"config_hash": fingerprint, "versions": versions} + temporary = metadata_path.with_suffix(".tmp") + temporary.write_text(json.dumps(metadata, indent=2)) + temporary.replace(metadata_path) + if tuning.search_only: + if not (tune_dir / "neps" / "pipeline_space.pkl").exists(): + raise ValueError("The primary process has not initialized NePS yet") + return tune_dir + + +def collect_trials(root: Path, *, wait: bool) -> pd.DataFrame: + """Wait for active evaluations and read all workers' trial records.""" + from neps.state import NePSState + + state = NePSState.create_or_load(root, load_only=True) + while True: + trials = state.lock_and_read_trials() + active = [t for t in trials.values() if t.metadata.state == "evaluating"] + if not wait or not active: + break + time.sleep(1) + records = [] + for trial in trials.values(): + if trial.metadata.state in {"success", "failed", "crashed"}: + records.append( + { + "neps_trial_id": trial.id, + "status": "failed", + **trial.report.extra, + } + ) + return pd.DataFrame(records) diff --git a/pyproject.toml b/pyproject.toml index 2f3586b3..a1887c56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,3 +113,6 @@ vllm = [ "transformers>=5.5.1,<6.0.0", ] llamacpp = ["llama-cpp-python>=0.3.0"] +tune = [ + "neural-pipeline-search==0.17.0", +] diff --git a/tests/test_tuning_config.py b/tests/test_tuning_config.py new file mode 100644 index 00000000..2de7d314 --- /dev/null +++ b/tests/test_tuning_config.py @@ -0,0 +1,91 @@ +"""Native NePS configuration and search-space contracts.""" + +import pytest +from pydantic import ValidationError + +from judgearena.config import TuneJudgeArgs +from judgearena.tuning.search_space import ( + FIDELITY, + apply_overrides, + axis_overrides, + build_neps_space, + decode_config, +) + + +def _settings(): + return TuneJudgeArgs( + neps={ + "optimizer": {"name": "neps_priorband", "eta": 3}, + "total_evaluations_to_spend": 10, + "pipeline_space": { + "judge.temperature": dict( + type="Float", + lower=0.0, + upper=1.0, + prior=0.2, + prior_confidence="medium", + ), + "judge.model": dict( + type="Categorical", + choices=["Dummy/judge", "Dummy/other"], + prior=1, + prior_confidence="medium", + ), + FIDELITY: dict(type="IntegerFidelity", lower=1, upper=3), + }, + }, + objectives=["agreement"], + price_per_million_tokens={"Dummy/judge": 1.0, "Dummy/other": 1.0}, + ) + + +def test_dotted_domains_and_index_priors(): + neps = pytest.importorskip("neps") + specs = _settings().neps["pipeline_space"] + specs["judge.max_out_tokens"] = dict(type="Integer", lower=16, upper=32) + attrs = build_neps_space(specs).get_attrs() + assert attrs["judge.model"].prior == 1 + assert attrs["judge.model"].choices == ("Dummy/judge", "Dummy/other") + assert isinstance(attrs["judge.temperature"], neps.Float) + assert attrs["judge.temperature"].prior == 0.2 + assert isinstance(attrs["judge.max_out_tokens"], neps.Integer) + assert isinstance(attrs[FIDELITY], neps.IntegerFidelity) + assert {"judge.temperature": 1.0} in list(axis_overrides(specs)) + overrides, battles = decode_config( + {"judge.temperature": 0.37, "judge.model": "Dummy/other", FIDELITY: 3} + ) + base = {"judge": {"temperature": 0.2}} + assert apply_overrides(base, overrides)["judge"] == { + "temperature": 0.37, + "model": "Dummy/other", + } + assert base["judge"]["temperature"] == 0.2 and battles == 3 + for name, spec in [ + ("meta_eval.split", {"type": "Categorical", "choices": ["test"]}), + ("judge.temperature", {"type": "IntegerFidelity", "lower": 1, "upper": 3}), + ]: + with pytest.raises(ValueError, match="tuner-owned|Only"): + build_neps_space({**specs, name: spec}) + + +@pytest.mark.parametrize( + "change,match", + [ + ({"neps": {"total_cost_to_spend": 1}}, "managed"), + ({"neps": {"root_directory": "elsewhere"}}, "managed"), + ({"neps": {}}, "finite global budget"), + ({"search_only": True, "run_dir": None}, "explicit"), + ({"neps": {"worker_evaluations_to_spend": 1}}, "Worker-local"), + ({"objectives": ["agreement", "cost_per_1k_battles"]}, "single objective"), + ({"neps": {"optimizer": {"name": "mo_hyperband"}}}, "agreement and cost"), + ({"price_per_million_tokens": {"Dummy/judge": -1}}, "nonnegative"), + ], +) +def test_tuning_config_validation(change, match): + values = _settings().model_dump() + if "neps" in change and change["neps"]: + change = {**change, "neps": {**values["neps"], **change["neps"]}} + values.update(change) + with pytest.raises(ValidationError, match=match): + TuneJudgeArgs(**values) From 64e95b2d78b3c8612ee9d32e3d31a4c2db42ffe6 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:30:40 +0200 Subject: [PATCH 04/13] Keep arena declarations unchanged in the split PR --- .../definitions/meta_eval/_comparia.yaml | 19 ------------------ .../definitions/meta_eval/_lmarena-100k.yaml | 19 ------------------ .../definitions/meta_eval/_lmarena-140k.yaml | 19 ------------------ .../meta_eval/meta-eval-comparia.yaml | 20 ++++++++++++++++++- .../meta_eval/meta-eval-lmarena-100k.yaml | 20 ++++++++++++++++++- .../meta_eval/meta-eval-lmarena-140k.yaml | 20 ++++++++++++++++++- 6 files changed, 57 insertions(+), 60 deletions(-) delete mode 100644 judgearena/tasks/definitions/meta_eval/_comparia.yaml delete mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml delete mode 100644 judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml diff --git a/judgearena/tasks/definitions/meta_eval/_comparia.yaml b/judgearena/tasks/definitions/meta_eval/_comparia.yaml deleted file mode 100644 index 6947fd5e..00000000 --- a/judgearena/tasks/definitions/meta_eval/_comparia.yaml +++ /dev/null @@ -1,19 +0,0 @@ -extends: _base.yaml - -variants: - selector: language - values: [fr, en] - -dataset: - sources: - comparia: - type: huggingface_dataset - repo_id: ministere-culture/comparia-votes - revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b" - allow_patterns: [votes.parquet] - -protocol: - arena: ComparIA - -metadata: - reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml deleted file mode 100644 index dafbf9c3..00000000 --- a/judgearena/tasks/definitions/meta_eval/_lmarena-100k.yaml +++ /dev/null @@ -1,19 +0,0 @@ -extends: _base.yaml - -variants: - selector: language - values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] - -dataset: - sources: - lmarena_100k: - type: huggingface_dataset - repo_id: lmarena-ai/arena-human-preference-100k - revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" - allow_patterns: [data/*.parquet] - -protocol: - arena: LMArena-100k - -metadata: - reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k diff --git a/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml deleted file mode 100644 index 36ea359e..00000000 --- a/judgearena/tasks/definitions/meta_eval/_lmarena-140k.yaml +++ /dev/null @@ -1,19 +0,0 @@ -extends: _base.yaml - -variants: - selector: language - values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] - -dataset: - sources: - lmarena_140k: - type: huggingface_dataset - repo_id: lmarena-ai/arena-human-preference-140k - revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74" - allow_patterns: [data/*.parquet] - -protocol: - arena: LMArena-140k - -metadata: - reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml index d0d35a00..f9a34f46 100644 --- a/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml +++ b/judgearena/tasks/definitions/meta_eval/meta-eval-comparia.yaml @@ -1,3 +1,21 @@ -extends: _comparia.yaml +extends: _base.yaml task: meta-eval-comparia description: Score a judge against human votes from the ComparIA arena. + +variants: + selector: language + values: [fr, en] + +dataset: + sources: + comparia: + type: huggingface_dataset + repo_id: ministere-culture/comparia-votes + revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b" + allow_patterns: [votes.parquet] + +protocol: + arena: ComparIA + +metadata: + reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml index 679d3db0..4fa21998 100644 --- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml +++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-100k.yaml @@ -1,3 +1,21 @@ -extends: _lmarena-100k.yaml +extends: _base.yaml task: meta-eval-lmarena-100k description: Score a judge against human votes from LMSYS Chatbot Arena 100k. + +variants: + selector: language + values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] + +dataset: + sources: + lmarena_100k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-100k + revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" + allow_patterns: [data/*.parquet] + +protocol: + arena: LMArena-100k + +metadata: + reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k diff --git a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml index 2ffddfd5..52ba8059 100644 --- a/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml +++ b/judgearena/tasks/definitions/meta_eval/meta-eval-lmarena-140k.yaml @@ -1,3 +1,21 @@ -extends: _lmarena-140k.yaml +extends: _base.yaml task: meta-eval-lmarena-140k description: Score a judge against human votes from LMSYS Chatbot Arena 140k. + +variants: + selector: language + values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] + +dataset: + sources: + lmarena_140k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-140k + revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74" + allow_patterns: [data/*.parquet] + +protocol: + arena: LMArena-140k + +metadata: + reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k From cb2091b2772a93989f286fe700cf1864caaa6869 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:31:44 +0200 Subject: [PATCH 05/13] Parameterize the generic judge tuning route by its meta-eval task --- judgearena/config.py | 25 ++++++++++++++++--------- judgearena/tasks/schema/__init__.py | 3 +-- judgearena/tasks/schema/meta_eval.py | 8 -------- judgearena/tasks/schema/task.py | 5 ++--- judgearena/tuning/__init__.py | 2 ++ 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/judgearena/config.py b/judgearena/config.py index 0719332d..b1ea81d9 100644 --- a/judgearena/config.py +++ b/judgearena/config.py @@ -26,8 +26,8 @@ EloScoringSpec, MetaEvalProtocol, MTBenchProtocol, - TuneJudgeProtocol, ) +from judgearena.tuning import TUNE_JUDGE_TASK # Set by build_run_config() for the duration of RunConfig() construction. _ACTIVE_CONFIG_PATH: str | None = None @@ -417,6 +417,9 @@ class TuneJudgeArgs(BaseModel): model_config = ConfigDict(use_attribute_docstrings=True, extra="forbid") + meta_eval_task: str + """Packaged meta-evaluation task used by every trial.""" + neps: dict[str, Any] """NePS constructor specs in pipeline_space and native neps.run arguments.""" @@ -557,14 +560,24 @@ def _validate(self) -> RunConfig: if not path.is_file(): raise ValueError(f"judge prompt file not found: {path}") - resolved_task = get_packaged_task(self.task) + is_tune_judge = self.task == TUNE_JUDGE_TASK + if (self.tune_judge is not None) != is_tune_judge: + raise ValueError( + "tune_judge config is required by, and only valid for, tune-judge." + ) + evaluation_task = self.tune_judge.meta_eval_task if is_tune_judge else self.task + resolved_task = get_packaged_task(evaluation_task) if resolved_task is None: raise ValueError( - f"Unknown task {self.task!r}; use 'judgearena tasks list' to " + f"Unknown task {evaluation_task!r}; use 'judgearena tasks list' to " "inspect packaged tasks." ) protocol = resolved_task.spec.protocol + if is_tune_judge and not isinstance(protocol, MetaEvalProtocol): + raise ValueError( + "tune_judge.meta_eval_task must select a meta-evaluation task." + ) task_generation = getattr(protocol, "generation", None) if ( "truncate_all_input_chars" not in self.generation.model_fields_set @@ -631,12 +644,6 @@ def _validate(self) -> RunConfig: raise ValueError( "meta_eval config is only valid for meta-evaluation tasks." ) - is_tune_judge = isinstance(protocol, TuneJudgeProtocol) - if (self.tune_judge is not None) != is_tune_judge: - raise ValueError( - "tune_judge config is required by, and only valid for, " - "tune-judge tasks." - ) if is_elo: if self.elo is None: self.elo = EloArgs() diff --git a/judgearena/tasks/schema/__init__.py b/judgearena/tasks/schema/__init__.py index 90039f45..2c82226f 100644 --- a/judgearena/tasks/schema/__init__.py +++ b/judgearena/tasks/schema/__init__.py @@ -15,7 +15,7 @@ ) from judgearena.tasks.schema.dataset import DatasetFields, DatasetSpec from judgearena.tasks.schema.elo import EloProtocol, EloScoringSpec -from judgearena.tasks.schema.meta_eval import MetaEvalProtocol, TuneJudgeProtocol +from judgearena.tasks.schema.meta_eval import MetaEvalProtocol from judgearena.tasks.schema.metrics import MetricSpec, ScoringSpec from judgearena.tasks.schema.mt_bench import ( MTBenchJudgeSpec, @@ -88,5 +88,4 @@ "TaskProvenance", "TaskSelection", "TaskSpec", - "TuneJudgeProtocol", ] diff --git a/judgearena/tasks/schema/meta_eval.py b/judgearena/tasks/schema/meta_eval.py index 6f0b52d6..3c01d120 100644 --- a/judgearena/tasks/schema/meta_eval.py +++ b/judgearena/tasks/schema/meta_eval.py @@ -39,11 +39,3 @@ def _validate_scoring(self) -> MetaEvalProtocol: f"meta-evaluation metric {request.metric} does not support breakdown_by" ) return self - - -class TuneJudgeProtocol(MetaEvalProtocol): - """Search judge settings by running trials of a meta-evaluation task.""" - - runner: Literal["tune_judge"] - meta_eval_task: str = Field(min_length=1) - """Task each trial runs as, so trials share its judgement cache.""" diff --git a/judgearena/tasks/schema/task.py b/judgearena/tasks/schema/task.py index a9f6e61e..6f5ab80e 100644 --- a/judgearena/tasks/schema/task.py +++ b/judgearena/tasks/schema/task.py @@ -13,7 +13,7 @@ ) from judgearena.tasks.schema.dataset import DatasetSpec from judgearena.tasks.schema.elo import EloProtocol -from judgearena.tasks.schema.meta_eval import MetaEvalProtocol, TuneJudgeProtocol +from judgearena.tasks.schema.meta_eval import MetaEvalProtocol from judgearena.tasks.schema.mt_bench import MTBenchProtocol from judgearena.tasks.schema.mt_bench_101 import MTBench101Protocol from judgearena.tasks.schema.pairwise import PairwiseProtocol @@ -23,8 +23,7 @@ | MTBenchProtocol | MTBench101Protocol | EloProtocol - | MetaEvalProtocol - | TuneJudgeProtocol, + | MetaEvalProtocol, Field(discriminator="runner"), ] diff --git a/judgearena/tuning/__init__.py b/judgearena/tuning/__init__.py index a2fa5b7b..21daed98 100644 --- a/judgearena/tuning/__init__.py +++ b/judgearena/tuning/__init__.py @@ -1 +1,3 @@ """Judge configuration tuning against human-labeled arena battles.""" + +TUNE_JUDGE_TASK = "tune-judge" From 842502240240baad1b793c64053253875ac98c8a Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:34:24 +0200 Subject: [PATCH 06/13] Supply the benchmark target in tuning configuration fixtures --- tests/test_tuning_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_tuning_config.py b/tests/test_tuning_config.py index 2de7d314..4f666778 100644 --- a/tests/test_tuning_config.py +++ b/tests/test_tuning_config.py @@ -15,6 +15,7 @@ def _settings(): return TuneJudgeArgs( + meta_eval_task="meta-eval-comparia-fr", neps={ "optimizer": {"name": "neps_priorband", "eta": 3}, "total_evaluations_to_spend": 10, From 8fb943ca35c6c8b973270ed31a5752ca896e7cd8 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:28:41 +0200 Subject: [PATCH 07/13] Package judge tuning defaults and simplify nested search configuration --- judgearena/benchmarks/registry.py | 7 ++ judgearena/config.py | 111 ++++++++++++------ .../tasks/definitions/tuning/tune-judge.yaml | 15 +++ judgearena/tasks/registry.py | 5 + judgearena/tasks/schema/__init__.py | 2 + judgearena/tasks/schema/task.py | 12 +- judgearena/tasks/schema/tuning.py | 15 +++ judgearena/tuning/search_space.py | 52 ++++++++ judgearena/utils/io.py | 7 +- tests/test_tuning_config.py | 92 +++++++++------ 10 files changed, 238 insertions(+), 80 deletions(-) create mode 100644 judgearena/tasks/definitions/tuning/tune-judge.yaml create mode 100644 judgearena/tasks/schema/tuning.py diff --git a/judgearena/benchmarks/registry.py b/judgearena/benchmarks/registry.py index 7bab0b06..abcce555 100644 --- a/judgearena/benchmarks/registry.py +++ b/judgearena/benchmarks/registry.py @@ -42,6 +42,12 @@ class ResolvedBenchmark: task: ResolvedTaskSpec | None +def _run_tune_judge(cfg, task): + from judgearena.tuning.runner import run_tune_judge + + return run_tune_judge(cfg, task) + + def benchmark_adapters() -> tuple[BenchmarkAdapter, ...]: """Return registered benchmark implementations, specific first.""" from judgearena.benchmarks.elo.runner import run_elo @@ -55,6 +61,7 @@ def benchmark_adapters() -> tuple[BenchmarkAdapter, ...]: BenchmarkAdapter("meta_eval", frozenset(), run_meta_eval), BenchmarkAdapter("mt_bench", frozenset(), run_mt_bench_benchmark), BenchmarkAdapter("mt_bench_101", frozenset(), run_mt_bench_101_benchmark), + BenchmarkAdapter("tune_judge", frozenset(), _run_tune_judge), BenchmarkAdapter("pairwise", None, run_pairwise), ) diff --git a/judgearena/config.py b/judgearena/config.py index b1ea81d9..a0f12522 100644 --- a/judgearena/config.py +++ b/judgearena/config.py @@ -26,8 +26,8 @@ EloScoringSpec, MetaEvalProtocol, MTBenchProtocol, + TuneJudgeProtocol, ) -from judgearena.tuning import TUNE_JUDGE_TASK # Set by build_run_config() for the duration of RunConfig() construction. _ACTIVE_CONFIG_PATH: str | None = None @@ -417,19 +417,54 @@ class TuneJudgeArgs(BaseModel): model_config = ConfigDict(use_attribute_docstrings=True, extra="forbid") - meta_eval_task: str + meta_eval_task: str | None = None """Packaged meta-evaluation task used by every trial.""" - neps: dict[str, Any] - """NePS constructor specs in pipeline_space and native neps.run arguments.""" + search_space: dict[str, Any] + """Nested judge parameters, written as lists or bounded ranges.""" - objectives: list[Literal["agreement", "cost_per_1k_battles"]] = Field( - default=["agreement", "cost_per_1k_battles"], min_length=1 + fidelity: dict[str, dict[str, int]] = Field(default_factory=dict) + optimizer: dict[str, Any] = Field(default_factory=dict) + neps: dict[str, Any] = Field(default_factory=dict) + """Native neps.run arguments; search space and optimizer are separate.""" + + objectives: list[Literal["agreement", "cost"]] | None = None + """Agreement is maximized and expected token cost minimized.""" + + price_per_million_tokens: dict[str, float | dict[str, float]] = Field( + default_factory=dict ) - """Ordered objectives; agreement is maximized and expected cost minimized.""" + """Optional USD per million tokens; scalar or separate input/output rates.""" - price_per_million_tokens: dict[str, float] - """USD per million judge tokens, including cached judgements.""" + def resolve(self, protocol: TuneJudgeProtocol) -> TuneJudgeArgs: + values = self.model_dump() + values["meta_eval_task"] = ( + self.meta_eval_task or protocol.default_meta_eval_task + ) + values["objectives"] = ( + self.objectives + if self.objectives is not None + else protocol.default_objectives + ) + values["fidelity"] = { + name: {**bounds, **self.fidelity.get(name, {})} + for name, bounds in protocol.default_fidelity.items() + } + if set(self.fidelity) - {"battles_per_model"}: + raise ValueError("Only battles_per_model fidelity is supported") + lower, upper = ( + values["fidelity"]["battles_per_model"][k] for k in ("lower", "upper") + ) + if not 0 < lower < upper: + raise ValueError("Fidelity requires 0 < lower < upper") + defaults = protocol.default_optimizer + values["optimizer"] = ( + {**defaults, **self.optimizer} + if self.optimizer.get("name", defaults["name"]) == defaults["name"] + else self.optimizer + ) + values["neps"] = {"ignore_errors": True, **self.neps} + return TuneJudgeArgs.model_validate(values) test_battles_per_model: int | None = Field(default=None, gt=0) """Test budget; defaults to the highest fidelity.""" @@ -443,6 +478,8 @@ class TuneJudgeArgs(BaseModel): @model_validator(mode="after") def _validate_runtime(self) -> TuneJudgeArgs: reserved = { + "pipeline_space", + "optimizer", "evaluate_pipeline", "root_directory", "overwrite_root_directory", @@ -467,22 +504,21 @@ def _validate_runtime(self) -> TuneJudgeArgs: for k in ("worker_evaluations_to_spend", "worker_fidelities_to_spend") ): raise ValueError("Worker-local budgets require search_only") - optimizer = self.neps.get("optimizer", "auto") - name = ( - optimizer.get("name") - if isinstance(optimizer, dict) - else optimizer[0] - if isinstance(optimizer, (list, tuple)) - else optimizer - ) - if name in {"neps_priorband", "neps_hyperband"} and len(self.objectives) != 1: - raise ValueError(f"{name} requires a single objective") - if name in {"mo_hyperband", "primo"} and len(self.objectives) != 2: - raise ValueError(f"{name} requires agreement and cost objectives") - if len(set(self.objectives)) != len(self.objectives): - raise ValueError("objectives must not contain duplicates") - if any(price < 0 for price in self.price_per_million_tokens.values()): - raise ValueError("Judge token prices must be nonnegative") + name = self.optimizer.get("name") + if self.objectives is not None: + if ( + name in {"neps_priorband", "neps_hyperband"} + and len(self.objectives) != 1 + ): + raise ValueError(f"{name} requires a single objective") + if name in {"mo_hyperband", "primo"} and len(self.objectives) != 2: + raise ValueError(f"{name} requires agreement and cost objectives") + if len(set(self.objectives)) != len(self.objectives): + raise ValueError("objectives must not contain duplicates") + for price in self.price_per_million_tokens.values(): + rates = price.values() if isinstance(price, dict) else [price] + if any(rate < 0 for rate in rates): + raise ValueError("Judge token prices must be nonnegative") return self @@ -560,24 +596,25 @@ def _validate(self) -> RunConfig: if not path.is_file(): raise ValueError(f"judge prompt file not found: {path}") - is_tune_judge = self.task == TUNE_JUDGE_TASK - if (self.tune_judge is not None) != is_tune_judge: - raise ValueError( - "tune_judge config is required by, and only valid for, tune-judge." - ) - evaluation_task = self.tune_judge.meta_eval_task if is_tune_judge else self.task - resolved_task = get_packaged_task(evaluation_task) + resolved_task = get_packaged_task(self.task) if resolved_task is None: raise ValueError( - f"Unknown task {evaluation_task!r}; use 'judgearena tasks list' to " - "inspect packaged tasks." + f"Unknown task {self.task!r}; use 'judgearena tasks list'." ) - protocol = resolved_task.spec.protocol - if is_tune_judge and not isinstance(protocol, MetaEvalProtocol): + is_tune_judge = isinstance(protocol, TuneJudgeProtocol) + if (self.tune_judge is not None) != is_tune_judge: raise ValueError( - "tune_judge.meta_eval_task must select a meta-evaluation task." + "tune_judge config is required by, and only valid for, tune-judge." ) + if is_tune_judge: + self.tune_judge = self.tune_judge.resolve(protocol) + target = get_packaged_task(self.tune_judge.meta_eval_task) + if target is None or not isinstance(target.spec.protocol, MetaEvalProtocol): + raise ValueError( + "tune_judge.meta_eval_task must select a meta-evaluation task." + ) + protocol = target.spec.protocol task_generation = getattr(protocol, "generation", None) if ( "truncate_all_input_chars" not in self.generation.model_fields_set diff --git a/judgearena/tasks/definitions/tuning/tune-judge.yaml b/judgearena/tasks/definitions/tuning/tune-judge.yaml new file mode 100644 index 00000000..aaa86c11 --- /dev/null +++ b/judgearena/tasks/definitions/tuning/tune-judge.yaml @@ -0,0 +1,15 @@ +schema_version: 1 +task: tune-judge +task_version: 1 +description: Tune a judge against human preferences with agreement and cost objectives. +tags: [tuning, judge] +protocol: + runner: tune_judge + default_meta_eval_task: meta-eval-lmarena-140k-en + default_objectives: [agreement, cost] + default_fidelity: + battles_per_model: {lower: 10, upper: 90} + default_optimizer: {name: mo_hyperband, eta: 3, mo_selector: epsnet} +metadata: + reference_implementation: https://github.com/geoalgo/judgetuning + paper: https://arxiv.org/abs/2501.17178 diff --git a/judgearena/tasks/registry.py b/judgearena/tasks/registry.py index 11e8a783..95c60aad 100644 --- a/judgearena/tasks/registry.py +++ b/judgearena/tasks/registry.py @@ -28,6 +28,7 @@ TaskProvenance, TaskSelection, TaskSpec, + TuneJudgeProtocol, ) logger = get_logger(__name__) @@ -317,6 +318,10 @@ def _discover_tasks( def _validate_adapter_ids(resolved: ResolvedTaskSpec, adapters: AdapterCatalog) -> None: spec = resolved.spec + if isinstance(spec.protocol, TuneJudgeProtocol): + if spec.protocol.runner not in adapters.runners: + raise TaskDefinitionError(f"Unknown runner {spec.protocol.runner!r}") + return is_battle_backed = isinstance(spec.protocol, (EloProtocol, MetaEvalProtocol)) dataset_names = ( adapters.battle_datasets if is_battle_backed else adapters.instruction_datasets diff --git a/judgearena/tasks/schema/__init__.py b/judgearena/tasks/schema/__init__.py index 2c82226f..b219511c 100644 --- a/judgearena/tasks/schema/__init__.py +++ b/judgearena/tasks/schema/__init__.py @@ -51,6 +51,7 @@ TaskMetadata, TaskSpec, ) +from judgearena.tasks.schema.tuning import TuneJudgeProtocol __all__ = [ "BaselineSpec", @@ -88,4 +89,5 @@ "TaskProvenance", "TaskSelection", "TaskSpec", + "TuneJudgeProtocol", ] diff --git a/judgearena/tasks/schema/task.py b/judgearena/tasks/schema/task.py index 6f5ab80e..c45fa79f 100644 --- a/judgearena/tasks/schema/task.py +++ b/judgearena/tasks/schema/task.py @@ -17,13 +17,15 @@ from judgearena.tasks.schema.mt_bench import MTBenchProtocol from judgearena.tasks.schema.mt_bench_101 import MTBench101Protocol from judgearena.tasks.schema.pairwise import PairwiseProtocol +from judgearena.tasks.schema.tuning import TuneJudgeProtocol ProtocolSpec = Annotated[ PairwiseProtocol | MTBenchProtocol | MTBench101Protocol | EloProtocol - | MetaEvalProtocol, + | MetaEvalProtocol + | TuneJudgeProtocol, Field(discriminator="runner"), ] @@ -73,7 +75,7 @@ class TaskSpec(StrictFrozenModel): task_version: int = Field(ge=1) description: str = Field(min_length=1) tags: tuple[str, ...] = () - dataset: DatasetSpec + dataset: DatasetSpec | None = None protocol: ProtocolSpec variants: SuffixVariants | None = None metadata: TaskMetadata = Field(default_factory=TaskMetadata) @@ -82,6 +84,12 @@ class TaskSpec(StrictFrozenModel): def _validate_task(self) -> TaskSpec: if len(set(self.tags)) != len(self.tags): raise ValueError("tags must not contain duplicates") + if isinstance(self.protocol, TuneJudgeProtocol): + if self.dataset is not None: + raise ValueError("Tuning tasks use their target benchmark's dataset.") + return self + if self.dataset is None: + raise ValueError("Benchmark tasks require a dataset.") source_names = set(self.dataset.sources) baseline = self.protocol.baseline if ( diff --git a/judgearena/tasks/schema/tuning.py b/judgearena/tasks/schema/tuning.py new file mode 100644 index 00000000..7f0604e7 --- /dev/null +++ b/judgearena/tasks/schema/tuning.py @@ -0,0 +1,15 @@ +"""Task-owned defaults for judge configuration search.""" + +from typing import Any, Literal + +from pydantic import Field + +from judgearena.tasks.schema.base import StrictFrozenModel + + +class TuneJudgeProtocol(StrictFrozenModel): + runner: Literal["tune_judge"] + default_meta_eval_task: str + default_objectives: list[Literal["agreement", "cost"]] + default_fidelity: dict[str, dict[str, int]] + default_optimizer: dict[str, Any] = Field(min_length=1) diff --git a/judgearena/tuning/search_space.py b/judgearena/tuning/search_space.py index 3ce6720b..8f097c03 100644 --- a/judgearena/tuning/search_space.py +++ b/judgearena/tuning/search_space.py @@ -13,6 +13,58 @@ DOMAIN_TYPES = frozenset({"Categorical", "Float", "Integer", "IntegerFidelity"}) +def parameter_specs(search_space: Mapping, base: Mapping, fidelity: Mapping) -> dict: + """Flatten nested parameter definitions and resolve value-based priors.""" + specs = {} + + def visit(section, prefix=""): + for key, value in section.items(): + name = f"{prefix}.{key}" if prefix else key + if isinstance(value, Mapping) and not ( + {"choices", "lower", "upper"} & value.keys() + ): + visit(value, name) + continue + if not ( + name.startswith("judge.") + or name == "generation.truncate_judge_input_chars" + ): + raise ValueError( + f"Search space must not override tuner-owned keys: {name}" + ) + spec = {"choices": value} if isinstance(value, list) else dict(value) + node = base + for part in name.split("."): + node = node.get(part) if isinstance(node, Mapping) else None + explicit_prior = "prior" in spec + prior = spec.pop("prior", node) + confidence = spec.pop("prior_confidence", "medium") + if "choices" in spec: + kind = "Categorical" + if explicit_prior and prior not in spec["choices"]: + raise ValueError(f"Prior for {name} is not in its choices") + prior = ( + spec["choices"].index(prior) if prior in spec["choices"] else None + ) + else: + kind = ( + "Integer" + if all(type(spec[k]) is int for k in ("lower", "upper")) + else "Float" + ) + if prior is not None and not spec["lower"] <= prior <= spec["upper"]: + if explicit_prior: + raise ValueError(f"Prior for {name} is outside its range") + prior = None + if prior is not None: + spec.update(prior=prior, prior_confidence=confidence) + specs[name] = {"type": kind, **spec} + + visit(search_space) + specs[FIDELITY] = {"type": "IntegerFidelity", **fidelity["battles_per_model"]} + return specs + + def build_neps_space(specs: Mapping[str, dict]) -> neps.PipelineSpace: """Translate constructor tags to domains without changing their arguments.""" import neps diff --git a/judgearena/utils/io.py b/judgearena/utils/io.py index aeab573e..6dba1198 100644 --- a/judgearena/utils/io.py +++ b/judgearena/utils/io.py @@ -29,6 +29,8 @@ def download_hf(name: str, local_path: Path): resolved_task = get_packaged_task(name) if resolved_task is None: raise ValueError(f"Unknown task {name!r}.") + if resolved_task.spec.dataset is None: + return from judgearena.datasets.registry import resolve_download_adapter resolve_download_adapter(resolved_task.spec.dataset.adapter).download( @@ -67,8 +69,9 @@ def download_all(): logger.info("Downloading all datasets in %s", data_root) local_path_tables = data_root / "tables" - for task_id in load_tasks(): - download_hf(name=task_id, local_path=local_path_tables) + for task_id, task in load_tasks().items(): + if task.spec.dataset is not None: + download_hf(name=task_id, local_path=local_path_tables) if __name__ == "__main__": diff --git a/tests/test_tuning_config.py b/tests/test_tuning_config.py index 4f666778..6429a0c3 100644 --- a/tests/test_tuning_config.py +++ b/tests/test_tuning_config.py @@ -15,59 +15,73 @@ def _settings(): return TuneJudgeArgs( - meta_eval_task="meta-eval-comparia-fr", - neps={ - "optimizer": {"name": "neps_priorband", "eta": 3}, - "total_evaluations_to_spend": 10, - "pipeline_space": { - "judge.temperature": dict( - type="Float", - lower=0.0, - upper=1.0, - prior=0.2, - prior_confidence="medium", - ), - "judge.model": dict( - type="Categorical", - choices=["Dummy/judge", "Dummy/other"], - prior=1, - prior_confidence="medium", - ), - FIDELITY: dict(type="IntegerFidelity", lower=1, upper=3), - }, + search_space={ + "judge": { + "temperature": {"lower": 0.0, "upper": 1.0, "prior_confidence": "low"}, + "model": { + "choices": ["Dummy/judge", "Dummy/other"], + "prior": "Dummy/other", + }, + "max_out_tokens": {"lower": 16, "upper": 32, "log": True}, + } }, + optimizer={"name": "neps_priorband", "eta": 3}, + fidelity={"battles_per_model": {"lower": 1, "upper": 3}}, + neps={"total_evaluations_to_spend": 10}, objectives=["agreement"], - price_per_million_tokens={"Dummy/judge": 1.0, "Dummy/other": 1.0}, ) -def test_dotted_domains_and_index_priors(): +def test_nested_domains_and_value_priors(): + from judgearena.tuning.search_space import parameter_specs + neps = pytest.importorskip("neps") - specs = _settings().neps["pipeline_space"] - specs["judge.max_out_tokens"] = dict(type="Integer", lower=16, upper=32) + settings = _settings() + specs = parameter_specs( + settings.search_space, {"judge": {"temperature": 0.2}}, settings.fidelity + ) attrs = build_neps_space(specs).get_attrs() assert attrs["judge.model"].prior == 1 assert attrs["judge.model"].choices == ("Dummy/judge", "Dummy/other") assert isinstance(attrs["judge.temperature"], neps.Float) assert attrs["judge.temperature"].prior == 0.2 assert isinstance(attrs["judge.max_out_tokens"], neps.Integer) + assert attrs["judge.max_out_tokens"].log assert isinstance(attrs[FIDELITY], neps.IntegerFidelity) assert {"judge.temperature": 1.0} in list(axis_overrides(specs)) - overrides, battles = decode_config( - {"judge.temperature": 0.37, "judge.model": "Dummy/other", FIDELITY: 3} - ) + overrides, battles = decode_config({"judge.temperature": 0.37, FIDELITY: 3}) base = {"judge": {"temperature": 0.2}} - assert apply_overrides(base, overrides)["judge"] == { - "temperature": 0.37, - "model": "Dummy/other", - } + assert apply_overrides(base, overrides)["judge"]["temperature"] == 0.37 assert base["judge"]["temperature"] == 0.2 and battles == 3 - for name, spec in [ - ("meta_eval.split", {"type": "Categorical", "choices": ["test"]}), - ("judge.temperature", {"type": "IntegerFidelity", "lower": 1, "upper": 3}), - ]: - with pytest.raises(ValueError, match="tuner-owned|Only"): - build_neps_space({**specs, name: spec}) + with pytest.raises(ValueError, match="tuner-owned"): + parameter_specs({"meta_eval": {"split": ["test"]}}, base, settings.fidelity) + + +def test_task_defaults_and_partial_overrides(): + from judgearena.config import RunConfig + from judgearena.tasks.registry import get_packaged_task + + task = get_packaged_task("tune-judge") + assert task.spec.dataset is None + cfg = RunConfig( + task="tune-judge", + judge={"model": "Dummy/judge"}, + tune_judge={ + "search_space": {"judge": {"temperature": [0.0, 1.0]}}, + "fidelity": {"battles_per_model": {"lower": 30}}, + "optimizer": {"eta": 2}, + "neps": {"total_evaluations_to_spend": 6}, + }, + ) + assert cfg.tune_judge.meta_eval_task == "meta-eval-lmarena-140k-en" + assert cfg.tune_judge.objectives == ["agreement", "cost"] + assert cfg.tune_judge.fidelity == {"battles_per_model": {"lower": 30, "upper": 90}} + assert cfg.tune_judge.optimizer == { + "name": "mo_hyperband", + "eta": 2, + "mo_selector": "epsnet", + } + assert cfg.tune_judge.neps["ignore_errors"] is True @pytest.mark.parametrize( @@ -78,8 +92,8 @@ def test_dotted_domains_and_index_priors(): ({"neps": {}}, "finite global budget"), ({"search_only": True, "run_dir": None}, "explicit"), ({"neps": {"worker_evaluations_to_spend": 1}}, "Worker-local"), - ({"objectives": ["agreement", "cost_per_1k_battles"]}, "single objective"), - ({"neps": {"optimizer": {"name": "mo_hyperband"}}}, "agreement and cost"), + ({"objectives": ["agreement", "cost"]}, "single objective"), + ({"optimizer": {"name": "mo_hyperband"}}, "agreement and cost"), ({"price_per_million_tokens": {"Dummy/judge": -1}}, "nonnegative"), ], ) From a4b2d3e87e5753fa251728094a80c008db187074 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:32:26 +0200 Subject: [PATCH 08/13] Skip dataset-free workflows in download coverage --- judgearena/config.py | 2 ++ judgearena/tuning/__init__.py | 2 -- tests/test_utils.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/judgearena/config.py b/judgearena/config.py index a0f12522..524d0683 100644 --- a/judgearena/config.py +++ b/judgearena/config.py @@ -516,6 +516,8 @@ def _validate_runtime(self) -> TuneJudgeArgs: if len(set(self.objectives)) != len(self.objectives): raise ValueError("objectives must not contain duplicates") for price in self.price_per_million_tokens.values(): + if isinstance(price, dict) and set(price) != {"input", "output"}: + raise ValueError("Token price overrides require input and output rates") rates = price.values() if isinstance(price, dict) else [price] if any(rate < 0 for rate in rates): raise ValueError("Judge token prices must be nonnegative") diff --git a/judgearena/tuning/__init__.py b/judgearena/tuning/__init__.py index 21daed98..a2fa5b7b 100644 --- a/judgearena/tuning/__init__.py +++ b/judgearena/tuning/__init__.py @@ -1,3 +1 @@ """Judge configuration tuning against human-labeled arena battles.""" - -TUNE_JUDGE_TASK = "tune-judge" diff --git a/tests/test_utils.py b/tests/test_utils.py index 0c0b10b8..14faae80 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -41,7 +41,9 @@ def test_download_all_dispatches_registered_tasks(monkeypatch, tmp_path): ) utils_io.download_all() - assert [name for _, name, _ in calls] == list(load_tasks()) + assert [name for _, name, _ in calls] == [ + name for name, task in load_tasks().items() if task.spec.dataset is not None + ] assert {path for _, _, path in calls} == {tmp_path / "tables"} From 9c81feacb6b1df7d2fe5ba486fc708fe46751bb6 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:39:38 +0200 Subject: [PATCH 09/13] Infer tuning range domains from config field types --- judgearena/tuning/search_space.py | 24 ++++++++++++++++++------ tests/test_tuning_config.py | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/judgearena/tuning/search_space.py b/judgearena/tuning/search_space.py index 8f097c03..71d2620f 100644 --- a/judgearena/tuning/search_space.py +++ b/judgearena/tuning/search_space.py @@ -4,7 +4,9 @@ import copy from collections.abc import Iterator, Mapping -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, get_args + +from judgearena.config import RunConfig if TYPE_CHECKING: import neps @@ -13,6 +15,20 @@ DOMAIN_TYPES = frozenset({"Categorical", "Float", "Integer", "IntegerFidelity"}) +def _range_type(name: str) -> str: + model = RunConfig + for part in name.split("."): + field = model.model_fields[part] + model = next( + t + for t in (get_args(field.annotation) or (field.annotation,)) + if t is not type(None) + ) + if model not in (int, float): + raise ValueError(f"Range requires a numeric config field: {name}") + return "Integer" if model is int else "Float" + + def parameter_specs(search_space: Mapping, base: Mapping, fidelity: Mapping) -> dict: """Flatten nested parameter definitions and resolve value-based priors.""" specs = {} @@ -47,11 +63,7 @@ def visit(section, prefix=""): spec["choices"].index(prior) if prior in spec["choices"] else None ) else: - kind = ( - "Integer" - if all(type(spec[k]) is int for k in ("lower", "upper")) - else "Float" - ) + kind = _range_type(name) if prior is not None and not spec["lower"] <= prior <= spec["upper"]: if explicit_prior: raise ValueError(f"Prior for {name} is outside its range") diff --git a/tests/test_tuning_config.py b/tests/test_tuning_config.py index 6429a0c3..f253177f 100644 --- a/tests/test_tuning_config.py +++ b/tests/test_tuning_config.py @@ -17,7 +17,7 @@ def _settings(): return TuneJudgeArgs( search_space={ "judge": { - "temperature": {"lower": 0.0, "upper": 1.0, "prior_confidence": "low"}, + "temperature": {"lower": 0, "upper": 1, "prior_confidence": "low"}, "model": { "choices": ["Dummy/judge", "Dummy/other"], "prior": "Dummy/other", From 66aa34af7585665ac9c2f76cc6a353273100dd2e Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:02:32 +0200 Subject: [PATCH 10/13] Apply suggestion from @ErlisLushtaku --- judgearena/tasks/schema/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/judgearena/tasks/schema/task.py b/judgearena/tasks/schema/task.py index c45fa79f..421fa2b3 100644 --- a/judgearena/tasks/schema/task.py +++ b/judgearena/tasks/schema/task.py @@ -86,7 +86,7 @@ def _validate_task(self) -> TaskSpec: raise ValueError("tags must not contain duplicates") if isinstance(self.protocol, TuneJudgeProtocol): if self.dataset is not None: - raise ValueError("Tuning tasks use their target benchmark's dataset.") + raise ValueError("Tuning tasks use their target meta-eval benchmark's dataset.") return self if self.dataset is None: raise ValueError("Benchmark tasks require a dataset.") From dfe71069e509f8b5f0f6d657330faa5ef3c47cac Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:21:03 +0200 Subject: [PATCH 11/13] Format the tuning task dataset validation message --- judgearena/tasks/schema/task.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/judgearena/tasks/schema/task.py b/judgearena/tasks/schema/task.py index 421fa2b3..d5a7a449 100644 --- a/judgearena/tasks/schema/task.py +++ b/judgearena/tasks/schema/task.py @@ -86,7 +86,9 @@ def _validate_task(self) -> TaskSpec: raise ValueError("tags must not contain duplicates") if isinstance(self.protocol, TuneJudgeProtocol): if self.dataset is not None: - raise ValueError("Tuning tasks use their target meta-eval benchmark's dataset.") + raise ValueError( + "Tuning tasks use their target meta-eval benchmark's dataset." + ) return self if self.dataset is None: raise ValueError("Benchmark tasks require a dataset.") From 8c5ea13fbd5660083cafefb25e8eaab0d636b845 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:08:45 +0200 Subject: [PATCH 12/13] Keep representative tuning configuration contracts --- tests/test_tuning_config.py | 118 ++++++++++-------------------------- 1 file changed, 33 insertions(+), 85 deletions(-) diff --git a/tests/test_tuning_config.py b/tests/test_tuning_config.py index f253177f..fbd4f443 100644 --- a/tests/test_tuning_config.py +++ b/tests/test_tuning_config.py @@ -1,106 +1,54 @@ -"""Native NePS configuration and search-space contracts.""" - import pytest -from pydantic import ValidationError - -from judgearena.config import TuneJudgeArgs -from judgearena.tuning.search_space import ( - FIDELITY, - apply_overrides, - axis_overrides, - build_neps_space, - decode_config, -) - - -def _settings(): - return TuneJudgeArgs( - search_space={ - "judge": { - "temperature": {"lower": 0, "upper": 1, "prior_confidence": "low"}, - "model": { - "choices": ["Dummy/judge", "Dummy/other"], - "prior": "Dummy/other", - }, - "max_out_tokens": {"lower": 16, "upper": 32, "log": True}, - } - }, - optimizer={"name": "neps_priorband", "eta": 3}, - fidelity={"battles_per_model": {"lower": 1, "upper": 3}}, - neps={"total_evaluations_to_spend": 10}, - objectives=["agreement"], - ) +from judgearena.config import RunConfig +from judgearena.tuning.search_space import FIDELITY, build_neps_space, parameter_specs -def test_nested_domains_and_value_priors(): - from judgearena.tuning.search_space import parameter_specs +def test_nested_space_and_priors(): neps = pytest.importorskip("neps") - settings = _settings() + domains = { + "judge": { + "temperature": {"lower": 0.0, "upper": 1.0}, + "model": {"choices": ["Dummy/a", "Dummy/b"], "prior": "Dummy/b"}, + } + } specs = parameter_specs( - settings.search_space, {"judge": {"temperature": 0.2}}, settings.fidelity + domains, + {"judge": {"temperature": 0.2}}, + {"battles_per_model": {"lower": 1, "upper": 3}}, ) attrs = build_neps_space(specs).get_attrs() - assert attrs["judge.model"].prior == 1 - assert attrs["judge.model"].choices == ("Dummy/judge", "Dummy/other") assert isinstance(attrs["judge.temperature"], neps.Float) assert attrs["judge.temperature"].prior == 0.2 - assert isinstance(attrs["judge.max_out_tokens"], neps.Integer) - assert attrs["judge.max_out_tokens"].log + assert attrs["judge.model"].prior == 1 assert isinstance(attrs[FIDELITY], neps.IntegerFidelity) - assert {"judge.temperature": 1.0} in list(axis_overrides(specs)) - overrides, battles = decode_config({"judge.temperature": 0.37, FIDELITY: 3}) - base = {"judge": {"temperature": 0.2}} - assert apply_overrides(base, overrides)["judge"]["temperature"] == 0.37 - assert base["judge"]["temperature"] == 0.2 and battles == 3 with pytest.raises(ValueError, match="tuner-owned"): - parameter_specs({"meta_eval": {"split": ["test"]}}, base, settings.fidelity) + parameter_specs({"meta_eval": {"split": ["test"]}}, {}, {}) -def test_task_defaults_and_partial_overrides(): - from judgearena.config import RunConfig - from judgearena.tasks.registry import get_packaged_task - - task = get_packaged_task("tune-judge") - assert task.spec.dataset is None +def test_packaged_defaults_and_target_restrictions(): + settings = { + "search_space": {"judge": {"temperature": [0.0, 1.0]}}, + "optimizer": {"eta": 2}, + "neps": {"total_evaluations_to_spend": 6}, + } cfg = RunConfig( - task="tune-judge", - judge={"model": "Dummy/judge"}, - tune_judge={ - "search_space": {"judge": {"temperature": [0.0, 1.0]}}, - "fidelity": {"battles_per_model": {"lower": 30}}, - "optimizer": {"eta": 2}, - "neps": {"total_evaluations_to_spend": 6}, - }, + task="tune-judge", judge={"model": "Dummy/judge"}, tune_judge=settings ) - assert cfg.tune_judge.meta_eval_task == "meta-eval-lmarena-140k-en" - assert cfg.tune_judge.objectives == ["agreement", "cost"] - assert cfg.tune_judge.fidelity == {"battles_per_model": {"lower": 30, "upper": 90}} + direct = RunConfig( + task=cfg.tune_judge.meta_eval_task, judge={"model": "Dummy/judge"} + ) + assert (cfg.judge, cfg.meta_eval) == (direct.judge, direct.meta_eval) assert cfg.tune_judge.optimizer == { "name": "mo_hyperband", "eta": 2, "mo_selector": "epsnet", } - assert cfg.tune_judge.neps["ignore_errors"] is True - - -@pytest.mark.parametrize( - "change,match", - [ - ({"neps": {"total_cost_to_spend": 1}}, "managed"), - ({"neps": {"root_directory": "elsewhere"}}, "managed"), - ({"neps": {}}, "finite global budget"), - ({"search_only": True, "run_dir": None}, "explicit"), - ({"neps": {"worker_evaluations_to_spend": 1}}, "Worker-local"), - ({"objectives": ["agreement", "cost"]}, "single objective"), - ({"optimizer": {"name": "mo_hyperband"}}, "agreement and cost"), - ({"price_per_million_tokens": {"Dummy/judge": -1}}, "nonnegative"), - ], -) -def test_tuning_config_validation(change, match): - values = _settings().model_dump() - if "neps" in change and change["neps"]: - change = {**change, "neps": {**values["neps"], **change["neps"]}} - values.update(change) - with pytest.raises(ValidationError, match=match): - TuneJudgeArgs(**values) + assert cfg.tune_judge.objectives == ["agreement", "cost"] + assert cfg.tune_judge.fidelity == {"battles_per_model": {"lower": 10, "upper": 90}} + with pytest.raises(ValueError): + RunConfig( + task="tune-judge", + judge={"model": "Dummy/judge"}, + tune_judge={**settings, "meta_eval_task": "mt-bench"}, + ) From c0477d9a172fdfc7daf79b432e7d3627f6334c88 Mon Sep 17 00:00:00 2001 From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:09:58 +0200 Subject: [PATCH 13/13] Run CI for stacked tuning pull requests --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9cfa7cc..57196b63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,6 @@ on: push: branches: [ "main" ] pull_request: - branches: [ "main" ] permissions: contents: read