From 5fb9cb9c51d26bc0bef9836cb103c4f3bea2a7e3 Mon Sep 17 00:00:00 2001 From: bora kargi Date: Tue, 29 Sep 2026 15:18:44 +0200 Subject: [PATCH 1/5] feat: add task-aware offline downloads --- .../pairwise/scoring/alpaca_eval.py | 8 ++++ .../benchmarks/pairwise/scoring/arena_hard.py | 26 ++++++++---- judgearena/benchmarks/scoring.py | 8 ++++ judgearena/tasks/cli.py | 10 +++++ judgearena/utils/io.py | 14 ++----- tests/test_task_registry.py | 16 +++++++- tests/test_utils.py | 41 +++++++++++++++++++ 7 files changed, 105 insertions(+), 18 deletions(-) diff --git a/judgearena/benchmarks/pairwise/scoring/alpaca_eval.py b/judgearena/benchmarks/pairwise/scoring/alpaca_eval.py index da760e9b..1e228965 100644 --- a/judgearena/benchmarks/pairwise/scoring/alpaca_eval.py +++ b/judgearena/benchmarks/pairwise/scoring/alpaca_eval.py @@ -223,6 +223,14 @@ def __post_init__(self) -> None: ): raise ValueError("gamed_weight must be a finite non-negative number.") + def prefetch(self) -> None: + """Download and validate the configured calibration data.""" + _load_gamed_data( + self.calibration_repo_id, + self.calibration_filename, + self.calibration_revision, + ) + def calculate(self, battles: pd.DataFrame) -> dict[str, object]: if not battles["pref"].notna().any(): return {} diff --git a/judgearena/benchmarks/pairwise/scoring/arena_hard.py b/judgearena/benchmarks/pairwise/scoring/arena_hard.py index 8d7a3bb7..6259e327 100644 --- a/judgearena/benchmarks/pairwise/scoring/arena_hard.py +++ b/judgearena/benchmarks/pairwise/scoring/arena_hard.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import re from dataclasses import dataclass from functools import lru_cache @@ -11,6 +12,7 @@ import pandas as pd from scipy.special import expit +from judgearena.paths import data_root from judgearena.utils.eval import PrefSummary DECISIVE_WEIGHT = 3 @@ -98,21 +100,27 @@ def _validate_calibration_protocol( return judge_id +@lru_cache(maxsize=1) +def _style_encoding(): + import tiktoken + + cache_dir = os.environ.get("TIKTOKEN_CACHE_DIR") or str( + data_root / "tiktoken-cache" + ) + os.environ["TIKTOKEN_CACHE_DIR"] = cache_dir + os.makedirs(cache_dir, exist_ok=True) + return tiktoken.encoding_for_model("gpt-4o") + + @lru_cache(maxsize=4096) def _style_features(completion: str) -> np.ndarray: """Return Arena-Hard's token-length and Markdown style features.""" - import tiktoken - without_code = completion for block in _CODE_BLOCK.findall(completion): without_code = without_code.replace(block, "") return np.asarray( [ - len( - tiktoken.encoding_for_model("gpt-4o").encode( - completion, disallowed_special=() - ) - ), + len(_style_encoding().encode(completion, disallowed_special=())), sum( len(re.findall(rf"^#{{{level}}}\s", without_code, re.MULTILINE)) for level in range(1, 7) @@ -544,6 +552,10 @@ def render(result: dict[str, object]) -> str: class ArenaHardV20Metric: """Official Arena-Hard v2 category-scoped score.""" + def prefetch(self) -> None: + """Cache the tokenizer used by style-controlled scoring.""" + _style_encoding() + def calculate(self, battles: pd.DataFrame) -> dict[str, object]: result: dict[str, object] = { **_summarize_battles(battles, require_complete_pairs=True).to_dict(), diff --git a/judgearena/benchmarks/scoring.py b/judgearena/benchmarks/scoring.py index c2d5ec34..6d5b7af3 100644 --- a/judgearena/benchmarks/scoring.py +++ b/judgearena/benchmarks/scoring.py @@ -98,6 +98,14 @@ def build_metrics( return tuple(configured) +def prefetch_metrics(requests: Sequence[MetricRequest]) -> None: + """Download external files required by configured metrics.""" + for _, metric in build_metrics(requests): + prefetch = getattr(metric, "prefetch", None) + if prefetch is not None: + prefetch() + + def _group_value(value: object) -> object: if pd.isna(value): return None diff --git a/judgearena/tasks/cli.py b/judgearena/tasks/cli.py index 3d428fe3..7776df82 100644 --- a/judgearena/tasks/cli.py +++ b/judgearena/tasks/cli.py @@ -27,6 +27,9 @@ def _parser() -> argparse.ArgumentParser: validate = commands.add_parser("validate", help="Validate packaged tasks.") validate.add_argument("task", nargs="?") + + download = commands.add_parser("download", help="Download a packaged task.") + download.add_argument("task", nargs="+") return parser @@ -56,6 +59,13 @@ def run_task_command( print(f"Validated task {args.task!r}.") else: print(f"Validated {len(tasks)} task(s).") + elif args.command == "download": + selected = [_require(parser, tasks, name) for name in args.task] + from judgearena.utils.io import data_root, download_hf + + for task in selected: + download_hf(task.task, data_root / "tables") + print(f"Downloaded task {task.task!r}.") except TaskDefinitionError as exc: parser.error(str(exc)) diff --git a/judgearena/utils/io.py b/judgearena/utils/io.py index 4e49fde9..3c5499b8 100644 --- a/judgearena/utils/io.py +++ b/judgearena/utils/io.py @@ -11,20 +11,11 @@ import pandas as pd from judgearena.log import get_logger +from judgearena.paths import data_root logger = get_logger(__name__) -def _data_root_path() -> Path: - raw = os.environ.get("JUDGEARENA_DATA") or os.environ.get("OPENJURY_DATA") - if raw: - return Path(raw).expanduser() - return Path("~/judgearena-data/").expanduser() - - -data_root = _data_root_path() - - def download_hf(name: str, local_path: Path): """Download the HF sources a packaged task declares in its YAML.""" from judgearena.tasks.registry import get_packaged_task @@ -37,6 +28,9 @@ def download_hf(name: str, local_path: Path): resolve_download_adapter(resolved_task.spec.dataset.adapter).download( resolved_task, local_path ) + from judgearena.benchmarks.scoring import prefetch_metrics + + prefetch_metrics(resolved_task.spec.protocol.scoring.metrics) def read_df(filename: Path, **pandas_kwargs) -> pd.DataFrame: diff --git a/tests/test_task_registry.py b/tests/test_task_registry.py index a40e7a1c..814818c9 100644 --- a/tests/test_task_registry.py +++ b/tests/test_task_registry.py @@ -7,6 +7,7 @@ import pytest import yaml +import judgearena.utils.io as utils_io from judgearena import cli as cli_module from judgearena.tasks.cli import run_task_command from judgearena.tasks.registry import ( @@ -496,7 +497,9 @@ def test_unknown_task_lists_registered_tasks(tmp_path, capsys): assert "test-task" in capsys.readouterr().err -def test_task_commands_list_show_and_validate(tmp_path, capsys, caplog): +def test_task_commands_list_show_validate_and_download( + tmp_path, capsys, caplog, monkeypatch +): _write_family( tmp_path, family="example", @@ -516,6 +519,17 @@ def test_task_commands_list_show_and_validate(tmp_path, capsys, caplog): run_task_command(["validate"], tasks=tasks) assert "Validated 1 task(s)." in capsys.readouterr().out + downloads = [] + monkeypatch.setattr(utils_io, "data_root", tmp_path) + monkeypatch.setattr( + utils_io, + "download_hf", + lambda name, path: downloads.append((name, path)), + ) + run_task_command(["download", "test-task"], tasks=tasks) + assert downloads == [("test-task", tmp_path / "tables")] + assert "Downloaded task 'test-task'." in capsys.readouterr().out + def test_task_show_reports_resolved_selection(tmp_path, capsys): definition = _task_definition("family") diff --git a/tests/test_utils.py b/tests/test_utils.py index 0c0b10b8..663c65a4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -45,6 +45,47 @@ def test_download_all_dispatches_registered_tasks(monkeypatch, tmp_path): assert {path for _, _, path in calls} == {tmp_path / "tables"} +def test_download_hf_prefetches_task_scoring_assets(monkeypatch, tmp_path): + import judgearena.benchmarks.pairwise.scoring.alpaca_eval as alpaca_scoring + import judgearena.benchmarks.pairwise.scoring.arena_hard as arena_scoring + import judgearena.datasets.registry as dataset_registry + + downloads = [] + calibration = [] + tokenizer = [] + + class Adapter: + @staticmethod + def download(task, path): + downloads.append((task.task, path)) + + monkeypatch.setattr(dataset_registry, "resolve_download_adapter", lambda _: Adapter) + monkeypatch.setattr( + alpaca_scoring, + "_load_gamed_data", + lambda *args: calibration.append(args), + ) + monkeypatch.setattr( + arena_scoring, "_style_encoding", lambda: tokenizer.append(True) + ) + + utils_io.download_hf("alpaca-eval", tmp_path) + utils_io.download_hf("arena-hard-v2.0", tmp_path) + + assert downloads == [ + ("alpaca-eval", tmp_path), + ("arena-hard-v2.0", tmp_path), + ] + assert calibration == [ + ( + "tatsu-lab/alpaca_eval", + "df_gamed.csv", + "2edc6fad8be6b14ea7230aabfd08188da6b8b814", + ) + ] + assert tokenizer == [True] + + def test_strip_thinking_tags_removes_full_reasoning_block(): raw = ( "so let me think through this carefully\n\n" From cdf9a7f1a24cbe0e885ae6f7bca74cb60ee907e1 Mon Sep 17 00:00:00 2001 From: bora kargi Date: Wed, 30 Sep 2026 12:47:33 +0200 Subject: [PATCH 2/5] chore: make datasets optional --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3bde5efd..11cc2fe8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "datasets>=4.0.0", "fast-langdetect>=1.0.0", "huggingface-hub>=0.34.4", "langchain-core>=0.3.27", @@ -97,6 +96,7 @@ quote-style = "double" indent-style = "space" [project.optional-dependencies] +scripts = ["datasets>=4.0.0"] # vLLM 0.19.1 is the pinned judge/battle runtime on this branch. Its own # Requires-Dist is `transformers!=5.0.*,...,!=5.5.0,>=4.56.0`, so the # only sub-range >=5.x that vLLM 0.19.1 accepts is >=5.5.1 — hence the From 29f3091f16ca5df7ea287748528aecbaad29b620 Mon Sep 17 00:00:00 2001 From: bora kargi Date: Mon, 5 Oct 2026 12:11:20 +0200 Subject: [PATCH 3/5] fix: reuse prepared task data for offline runs --- judgearena/arenas_utils.py | 2 + judgearena/datasets/arena_battles.py | 2 + judgearena/datasets/fluency.py | 9 +- judgearena/datasets/judgearena_tables.py | 4 + judgearena/datasets/m_arenahard.py | 23 ++++- tests/test_elo_task_runtime.py | 25 +++++ tests/test_instruction_dataset.py | 123 +++++++++++++++++++++-- 7 files changed, 175 insertions(+), 13 deletions(-) diff --git a/judgearena/arenas_utils.py b/judgearena/arenas_utils.py index 8f117960..9a8a86ba 100644 --- a/judgearena/arenas_utils.py +++ b/judgearena/arenas_utils.py @@ -5,6 +5,7 @@ import pandas as pd from fast_langdetect import detect_language from huggingface_hub import snapshot_download +from huggingface_hub.constants import HF_HUB_OFFLINE from judgearena.log import get_logger from judgearena.tasks.schema import HuggingFaceDatasetSource @@ -31,6 +32,7 @@ def _download_arena_dataset( allow_patterns=source.allow_patterns or default_allow_patterns, force_download=False, revision=source.revision, + local_files_only=HF_HUB_OFFLINE, ) diff --git a/judgearena/datasets/arena_battles.py b/judgearena/datasets/arena_battles.py index dcfd9844..668d49db 100644 --- a/judgearena/datasets/arena_battles.py +++ b/judgearena/datasets/arena_battles.py @@ -6,6 +6,7 @@ import pandas as pd from huggingface_hub import snapshot_download +from huggingface_hub.constants import HF_HUB_OFFLINE from judgearena.arenas_utils import ( KNOWN_ARENAS, @@ -48,6 +49,7 @@ def download_task_sources(task: ResolvedTaskSpec, _local_dir: Path) -> None: revision=source.revision, allow_patterns=source.allow_patterns or None, force_download=False, + local_files_only=HF_HUB_OFFLINE, ) diff --git a/judgearena/datasets/fluency.py b/judgearena/datasets/fluency.py index 5d4a9d24..8f841dc3 100644 --- a/judgearena/datasets/fluency.py +++ b/judgearena/datasets/fluency.py @@ -13,6 +13,7 @@ import pandas as pd from huggingface_hub import snapshot_download +from huggingface_hub.constants import HF_HUB_OFFLINE from judgearena.tasks.schema import HuggingFaceDatasetSource, ResolvedTaskSpec @@ -97,12 +98,18 @@ def download_task_sources(task: ResolvedTaskSpec, local_tables_path: Path) -> No if task.spec.dataset.adapter != "fluency": raise ValueError(f"Task {task.task!r} does not use the fluency adapter.") source = _source(task) + local_dir = _source_local_dir(source, local_tables_path) + if HF_HUB_OFFLINE and all( + any(path.is_file() for path in (local_dir / language).glob("*.parquet")) + for language in _selected_languages(task) + ): + return snapshot_download( repo_id=source.repo_id, repo_type="dataset", revision=source.revision, allow_patterns=list(source.allow_patterns) or None, - local_dir=_source_local_dir(source, local_tables_path), + local_dir=local_dir, force_download=False, ) diff --git a/judgearena/datasets/judgearena_tables.py b/judgearena/datasets/judgearena_tables.py index 0bf40478..c7f2d354 100644 --- a/judgearena/datasets/judgearena_tables.py +++ b/judgearena/datasets/judgearena_tables.py @@ -49,6 +49,10 @@ def download_task_sources(task: ResolvedTaskSpec, local_dir: Path) -> None: f"Dataset source {name!r} for task {task.task!r} is not supported " "by the 'judgearena_tables' adapter." ) + if source.allow_patterns and all( + (local_dir / pattern).is_file() for pattern in source.allow_patterns + ): + continue snapshot_download( repo_id=source.repo_id, repo_type="dataset", diff --git a/judgearena/datasets/m_arenahard.py b/judgearena/datasets/m_arenahard.py index 6fc85532..dab271a3 100644 --- a/judgearena/datasets/m_arenahard.py +++ b/judgearena/datasets/m_arenahard.py @@ -6,6 +6,7 @@ import pandas as pd from huggingface_hub import snapshot_download +from huggingface_hub.constants import HF_HUB_OFFLINE from judgearena.tasks.registry import get_packaged_task from judgearena.tasks.schema import HuggingFaceDatasetSource, ResolvedTaskSpec @@ -57,6 +58,21 @@ def _download_source( if name == "examples" else local_tables_path ) + if ( + name == "examples" + and HF_HUB_OFFLINE + and all( + any( + path.is_file() for path in (local_dir / language).glob("test-*.parquet") + ) + for language in _selected_languages(task) + ) + ): + return + if name == "outputs" and task.selection is not None: + path = local_dir / "model_outputs" / f"{task.task}.csv.zip" + if path.is_file(): + return snapshot_download( repo_id=declared_source.repo_id, repo_type="dataset", @@ -87,6 +103,7 @@ def _load_source_frames( selected = set(_selected_languages(task)) frames: list[pd.DataFrame] = [] + found_languages: set[str] = set() for parquet_path in sorted(source_root.rglob("*.parquet")): language = parquet_path.parent.name if language not in selected: @@ -94,10 +111,12 @@ def _load_source_frames( frame = pd.read_parquet(parquet_path) frame["lang"] = language frames.append(frame) + found_languages.add(language) - if not frames: + missing = selected - found_languages + if missing: raise FileNotFoundError( - f"No m-ArenaHard parquet files for {sorted(selected)} under {source_root}." + f"No m-ArenaHard parquet files for {sorted(missing)} under {source_root}." ) df = pd.concat(frames, ignore_index=True) diff --git a/tests/test_elo_task_runtime.py b/tests/test_elo_task_runtime.py index 5f48af1d..8df6e411 100644 --- a/tests/test_elo_task_runtime.py +++ b/tests/test_elo_task_runtime.py @@ -2,6 +2,7 @@ import pandas as pd +import judgearena.arenas_utils as arenas_utils import judgearena.datasets.arena_battles as arena_battles from judgearena.tasks.registry import get_packaged_task @@ -36,6 +37,7 @@ def fake_load_arena_dataframe(arena, *, dataset_sources): def test_elo_dataset_download_uses_pinned_task_source(monkeypatch, tmp_path): task = _elo_task("elo-lmarena-100k") captured = {} + monkeypatch.setattr(arena_battles, "HF_HUB_OFFLINE", True) monkeypatch.setattr( arena_battles, "snapshot_download", @@ -47,3 +49,26 @@ def test_elo_dataset_download_uses_pinned_task_source(monkeypatch, tmp_path): assert captured["repo_id"] == "lmarena-ai/arena-human-preference-100k" assert captured["revision"] == "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" assert captured["allow_patterns"] == ("data/*.parquet",) + assert captured["local_files_only"] is True + + +def test_elo_runtime_uses_cached_snapshot_offline(monkeypatch): + task = _elo_task("elo-lmarena-100k") + repo_id = "lmarena-ai/arena-human-preference-100k" + captured = {} + monkeypatch.setattr(arenas_utils, "HF_HUB_OFFLINE", True) + + def cached_snapshot(**kwargs): + captured.update(kwargs) + return "/cached/snapshot" + + monkeypatch.setattr(arenas_utils, "snapshot_download", cached_snapshot) + result = arenas_utils._download_arena_dataset( + repo_id=repo_id, + default_allow_patterns="data/*.parquet", + dataset_sources={repo_id: task.spec.dataset.sources["lmarena_100k"]}, + ) + + assert result == "/cached/snapshot" + assert captured["revision"] == "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" + assert captured["local_files_only"] is True diff --git a/tests/test_instruction_dataset.py b/tests/test_instruction_dataset.py index 6c4606b7..945b81db 100644 --- a/tests/test_instruction_dataset.py +++ b/tests/test_instruction_dataset.py @@ -41,9 +41,27 @@ def test_table_adapter_download_uses_declared_paths(monkeypatch, tmp_path): assert captured["local_dir"] == tmp_path +def test_table_adapter_downloads_if_only_instruction_is_cached(monkeypatch, tmp_path): + task = get_packaged_task("alpaca-eval") + assert task is not None + path = tmp_path / "instructions" / "alpaca-eval.csv" + path.parent.mkdir() + path.touch() + calls = [] + monkeypatch.setattr( + judgearena_tables, "snapshot_download", lambda **kwargs: calls.append(kwargs) + ) + + judgearena_tables.download_task_sources(task, tmp_path) + + assert len(calls) == 1 + + def test_table_adapter_reuses_and_normalizes_alpaca_tables(monkeypatch, tmp_path): monkeypatch.setattr( - judgearena_tables, "download_task_sources", lambda _task, _path: None + judgearena_tables, + "snapshot_download", + lambda **kwargs: pytest.fail("Prepared tables must load without a Hub call"), ) instructions_dir = tmp_path / "instructions" outputs_dir = tmp_path / "model_outputs" @@ -129,20 +147,66 @@ def test_m_arena_hard_sources_and_baselines_are_owned_by_task_yaml(): assert v20.spec.protocol.baseline.reference_id == "google/gemini-2.5-flash" +@pytest.mark.parametrize("offline, expected_calls", [(True, 0), (False, 1)]) +def test_m_arena_hard_checks_selected_local_examples( + monkeypatch, tmp_path, offline, expected_calls +): + task = get_packaged_task("m-arena-hard-v0.1-uk") + assert task is not None + source = task.spec.dataset.sources["examples"] + path = tmp_path / "_sources" / source.repo_id.replace("/", "--") / "uk" + path.mkdir(parents=True) + (path / "test-00000.parquet").touch() + calls = [] + monkeypatch.setattr(m_arenahard, "HF_HUB_OFFLINE", offline) + monkeypatch.setattr( + m_arenahard, "snapshot_download", lambda **kwargs: calls.append(kwargs) + ) + + m_arenahard._download_source(task, "examples", tmp_path) + + assert len(calls) == expected_calls + + +def test_m_arena_hard_downloads_if_selected_language_is_missing(monkeypatch, tmp_path): + task = get_packaged_task("m-arena-hard-v2.0-EU") + assert task is not None + source = task.spec.dataset.sources["examples"] + root = tmp_path / "_sources" / source.repo_id.replace("/", "--") + (root / "cs").mkdir(parents=True) + (root / "cs" / "test-00000.parquet").touch() + monkeypatch.setattr(m_arenahard, "HF_HUB_OFFLINE", True) + calls = [] + monkeypatch.setattr( + m_arenahard, "snapshot_download", lambda **kwargs: calls.append(kwargs) + ) + + m_arenahard._download_source(task, "examples", tmp_path) + + assert len(calls) == 1 + monkeypatch.setattr( + m_arenahard.pd, + "read_parquet", + lambda path: pd.DataFrame({"question_id": ["q1"], "prompt": ["Czech"]}), + ) + with pytest.raises(FileNotFoundError, match="uk"): + m_arenahard.load_task_instructions(task, tmp_path) + + def test_m_arena_hard_adapter_filters_selected_language_group(monkeypatch, tmp_path): task = get_packaged_task("m-arena-hard-v2.0-EU") assert task is not None source = task.spec.dataset.sources["examples"] source_root = tmp_path / "_sources" / source.repo_id.replace("/", "--") - for language in ("cs", "uk", "ar"): + selected = m_arenahard._selected_languages(task) + for language in (*selected, "ar"): language_dir = source_root / language language_dir.mkdir(parents=True) (language_dir / "test.parquet").touch() frames = { - "cs": pd.DataFrame({"question_id": ["q1"], "prompt": ["Czech"]}), - "uk": pd.DataFrame({"question_id": ["q1"], "prompt": ["Ukrainian"]}), - "ar": pd.DataFrame({"question_id": ["q1"], "prompt": ["Arabic"]}), + language: pd.DataFrame({"question_id": ["q1"], "prompt": [language]}) + for language in (*selected, "ar") } monkeypatch.setattr( m_arenahard, "_download_source", lambda _task, _name, _path, **_kwargs: None @@ -155,9 +219,10 @@ def test_m_arena_hard_adapter_filters_selected_language_group(monkeypatch, tmp_p loaded = m_arenahard.load_task_instructions(task, tmp_path) - assert loaded["instruction_index"].tolist() == ["q1-cs", "q1-uk"] - assert loaded["instruction"].tolist() == ["Czech", "Ukrainian"] - assert loaded["lang"].tolist() == ["cs", "uk"] + assert loaded["instruction_index"].tolist() == [ + f"q1-{lang}" for lang in sorted(selected) + ] + assert loaded["lang"].tolist() == sorted(selected) def test_m_arena_hard_adapter_loads_invocation_specific_outputs(monkeypatch, tmp_path): @@ -174,7 +239,9 @@ def test_m_arena_hard_adapter_loads_invocation_specific_outputs(monkeypatch, tmp ) expected.to_csv(output_path, index=False) monkeypatch.setattr( - m_arenahard, "_download_source", lambda _task, _name, _path, **_kwargs: None + m_arenahard, + "snapshot_download", + lambda **kwargs: pytest.fail("Prepared data must load without a Hub call"), ) loaded = m_arenahard.load_task_model_outputs(task, tmp_path) @@ -480,10 +547,46 @@ def test_fluency_variants_cover_all_supported_languages(): assert set(task.spec.variants.values) == expected +def test_fluency_downloads_if_selected_language_is_missing(monkeypatch, tmp_path): + task = get_packaged_task("fluency-french") + assert task is not None + monkeypatch.setattr(fluency, "HF_HUB_OFFLINE", True) + calls = [] + monkeypatch.setattr( + fluency, "snapshot_download", lambda **kwargs: calls.append(kwargs) + ) + + fluency.download_task_sources(task, tmp_path) + + assert len(calls) == 1 + + +def test_fluency_online_download_completes_existing_data(monkeypatch, tmp_path): + task = get_packaged_task("fluency-french") + assert task is not None + root = fluency._source_local_dir(fluency._source(task), tmp_path) + (root / "French").mkdir(parents=True) + (root / "French" / "data.parquet").touch() + monkeypatch.setattr(fluency, "HF_HUB_OFFLINE", False) + calls = [] + monkeypatch.setattr( + fluency, "snapshot_download", lambda **kwargs: calls.append(kwargs) + ) + + fluency.download_task_sources(task, tmp_path) + + assert len(calls) == 1 + + def test_fluency_adapter_loads_selected_language(monkeypatch, tmp_path): task = get_packaged_task("fluency-french") assert task is not None - monkeypatch.setattr(fluency, "download_task_sources", lambda _task, _path: None) + monkeypatch.setattr(fluency, "HF_HUB_OFFLINE", True) + monkeypatch.setattr( + fluency, + "snapshot_download", + lambda **kwargs: pytest.fail("Prepared data must load without a Hub call"), + ) root = fluency._source_local_dir(fluency._source(task), tmp_path) (root / "French").mkdir(parents=True) pd.DataFrame({"sentence": ["Le chat", "La maison"]}).to_parquet( From d60e3cf1a98888953ca451ed6b2850c6aeeb0c3f Mon Sep 17 00:00:00 2001 From: bora kargi Date: Tue, 6 Oct 2026 15:48:26 +0200 Subject: [PATCH 4/5] fix: use task-owned MT-Bench cache by default --- judgearena/datasets/mt_bench.py | 10 +++++++--- judgearena/tasks/README.md | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/judgearena/datasets/mt_bench.py b/judgearena/datasets/mt_bench.py index 1eadf572..ea00c5f9 100644 --- a/judgearena/datasets/mt_bench.py +++ b/judgearena/datasets/mt_bench.py @@ -143,7 +143,10 @@ def _download_mt_bench( def download_mt_bench(local_dir: Path | None = None) -> tuple[Path, Path | None]: """Compatibility wrapper downloading the registered MT-Bench sources.""" - return _download_mt_bench(_task(), local_dir or data_root / "mt-bench") + task = _task() + return _download_mt_bench( + task, local_dir or _task_cache_dir(task, data_root / "tables") + ) def download_mt_bench_model_answer( @@ -154,7 +157,7 @@ def download_mt_bench_model_answer( ) -> Path: """Download a cached MT-Bench model-answer file if missing.""" resolved = task or _task() - root = local_dir or data_root / "mt-bench" + root = local_dir or _task_cache_dir(resolved, data_root / "tables") answer_path = root / "data" / "mt_bench" / "model_answer" / f"{model_id}.jsonl" if answer_path.exists(): return answer_path @@ -344,4 +347,5 @@ def load_task_model_outputs( def load_mt_bench() -> pd.DataFrame: """Compatibility wrapper loading the registered MT-Bench task.""" - return _load_mt_bench(_task(), data_root / "mt-bench") + task = _task() + return _load_mt_bench(task, _task_cache_dir(task, data_root / "tables")) diff --git a/judgearena/tasks/README.md b/judgearena/tasks/README.md index f2fe9578..0d30000a 100644 --- a/judgearena/tasks/README.md +++ b/judgearena/tasks/README.md @@ -26,6 +26,24 @@ Executable behavior lives outside the definitions: - `judgearena/benchmarks/` runs the selected evaluation workflow. - `judgearena/prompts/` owns reusable judge prompt presets. +## Task data + +Set `JUDGEARENA_DATA` before downloading and running a task. By default it is +`OPENJURY_DATA` or `~/judgearena-data`. Use the same path on the download host +and compute node, and bind it into the container. + +- `$JUDGEARENA_DATA/tables/`: task-owned inputs and archived outputs. MT-Bench + keeps its pinned questions and baseline answers under `tables/_sources/mt-bench/`. +- `$JUDGEARENA_DATA/cache/`: generated model responses. +- `HF_HOME`: separate Hugging Face model/cache storage. +- `--run.result_folder`: run results, separate from task inputs. + +`judgearena tasks download mt-bench` prepares its sources under `tables/`. +Use the same `JUDGEARENA_DATA` for evaluation. Do not move answers to the old +`$JUDGEARENA_DATA/mt-bench` path; re-download into the current task path instead. +Some task variants lack archived baseline answers and require generation from +the declared baseline model. + ## Add a task using existing components Create a public YAML file under `definitions//`. Files beginning with From 45e3b80705f54463f7b67bdeecd8245413eb7457 Mon Sep 17 00:00:00 2001 From: bora kargi Date: Tue, 6 Oct 2026 16:18:22 +0200 Subject: [PATCH 5/5] fix: declare parquet runtime dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5be1a60d..52c8bb64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "langchain-openai>=0.3.32", "langchain-together>=0.3.1", "pandas>=2.3.2", + "pyarrow", "pyyaml>=6.0.2", "scikit-learn>=1.8.0,<1.10", "tiktoken>=0.7.0",