Make the Tasks downloadable from the cli

judgearena/arenas_utils.py

@@ -5,6 +5,7 @@
 import pandas as pd
 from fast_langdetect import detect_language
 from huggingface_hub import snapshot_download
+from huggingface_hub.constants import HF_HUB_OFFLINE
 
 from judgearena.log import get_logger
 from judgearena.tasks.schema import HuggingFaceDatasetSource
@@ -31,6 +32,7 @@ def _download_arena_dataset(
         allow_patterns=source.allow_patterns or default_allow_patterns,
         force_download=False,
         revision=source.revision,
+        local_files_only=HF_HUB_OFFLINE,
     )
 
 

judgearena/benchmarks/pairwise/scoring/alpaca_eval.py

@@ -223,6 +223,14 @@ def __post_init__(self) -> None:
         ):
             raise ValueError("gamed_weight must be a finite non-negative number.")
 
+    def prefetch(self) -> None:
+        """Download and validate the configured calibration data."""
+        _load_gamed_data(
+            self.calibration_repo_id,
+            self.calibration_filename,
+            self.calibration_revision,
+        )
+
     def calculate(self, battles: pd.DataFrame) -> dict[str, object]:
         if not battles["pref"].notna().any():
             return {}

judgearena/benchmarks/pairwise/scoring/arena_hard.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 import re
 from dataclasses import dataclass
 from functools import lru_cache
@@ -11,6 +12,7 @@
 import pandas as pd
 from scipy.special import expit
 
+from judgearena.paths import data_root
 from judgearena.utils.eval import PrefSummary
 
 DECISIVE_WEIGHT = 3
@@ -98,21 +100,27 @@ def _validate_calibration_protocol(
     return judge_id
 
 
+@lru_cache(maxsize=1)
+def _style_encoding():
+    import tiktoken
+
+    cache_dir = os.environ.get("TIKTOKEN_CACHE_DIR") or str(
+        data_root / "tiktoken-cache"
+    )
+    os.environ["TIKTOKEN_CACHE_DIR"] = cache_dir
+    os.makedirs(cache_dir, exist_ok=True)
+    return tiktoken.encoding_for_model("gpt-4o")
+
+
 @lru_cache(maxsize=4096)
 def _style_features(completion: str) -> np.ndarray:
     """Return Arena-Hard's token-length and Markdown style features."""
-    import tiktoken
-
     without_code = completion
     for block in _CODE_BLOCK.findall(completion):
         without_code = without_code.replace(block, "")
     return np.asarray(
         [
-            len(
-                tiktoken.encoding_for_model("gpt-4o").encode(
-                    completion, disallowed_special=()
-                )
-            ),
+            len(_style_encoding().encode(completion, disallowed_special=())),
             sum(
                 len(re.findall(rf"^#{{{level}}}\s", without_code, re.MULTILINE))
                 for level in range(1, 7)
@@ -544,6 +552,10 @@ def render(result: dict[str, object]) -> str:
 class ArenaHardV20Metric:
     """Official Arena-Hard v2 category-scoped score."""
 
+    def prefetch(self) -> None:
+        """Cache the tokenizer used by style-controlled scoring."""
+        _style_encoding()
+
     def calculate(self, battles: pd.DataFrame) -> dict[str, object]:
         result: dict[str, object] = {
             **_summarize_battles(battles, require_complete_pairs=True).to_dict(),

judgearena/benchmarks/scoring.py

@@ -100,6 +100,14 @@ def build_metrics(
     return tuple(configured)
 
 
+def prefetch_metrics(requests: Sequence[MetricRequest]) -> None:
+    """Download external files required by configured metrics."""
+    for _, metric in build_metrics(requests):
+        prefetch = getattr(metric, "prefetch", None)
+        if prefetch is not None:
+            prefetch()
+
+
 def _group_value(value: object) -> object:
     if pd.isna(value):
         return None

judgearena/datasets/arena_battles.py

@@ -6,6 +6,7 @@
 
 import pandas as pd
 from huggingface_hub import snapshot_download
+from huggingface_hub.constants import HF_HUB_OFFLINE
 
 from judgearena.arenas_utils import (
     KNOWN_ARENAS,
@@ -48,6 +49,7 @@ def download_task_sources(task: ResolvedTaskSpec, _local_dir: Path) -> None:
             revision=source.revision,
             allow_patterns=source.allow_patterns or None,
             force_download=False,
+            local_files_only=HF_HUB_OFFLINE,
         )
 
 

judgearena/datasets/fluency.py

@@ -13,6 +13,7 @@
 
 import pandas as pd
 from huggingface_hub import snapshot_download
+from huggingface_hub.constants import HF_HUB_OFFLINE
 
 from judgearena.tasks.schema import HuggingFaceDatasetSource, ResolvedTaskSpec
 
@@ -97,12 +98,18 @@ def download_task_sources(task: ResolvedTaskSpec, local_tables_path: Path) -> No
     if task.spec.dataset.adapter != "fluency":
         raise ValueError(f"Task {task.task!r} does not use the fluency adapter.")
     source = _source(task)
+    local_dir = _source_local_dir(source, local_tables_path)
+    if HF_HUB_OFFLINE and all(
+        any(path.is_file() for path in (local_dir / language).glob("*.parquet"))
+        for language in _selected_languages(task)
+    ):
+        return
     snapshot_download(
         repo_id=source.repo_id,
         repo_type="dataset",
         revision=source.revision,
         allow_patterns=list(source.allow_patterns) or None,
-        local_dir=_source_local_dir(source, local_tables_path),
+        local_dir=local_dir,
         force_download=False,
     )
 

judgearena/datasets/judgearena_tables.py

@@ -49,6 +49,10 @@ def download_task_sources(task: ResolvedTaskSpec, local_dir: Path) -> None:
                 f"Dataset source {name!r} for task {task.task!r} is not supported "
                 "by the 'judgearena_tables' adapter."
             )
+        if source.allow_patterns and all(
+            (local_dir / pattern).is_file() for pattern in source.allow_patterns
+        ):
+            continue
         snapshot_download(
             repo_id=source.repo_id,
             repo_type="dataset",

judgearena/datasets/m_arenahard.py

@@ -6,6 +6,7 @@
 
 import pandas as pd
 from huggingface_hub import snapshot_download
+from huggingface_hub.constants import HF_HUB_OFFLINE
 
 from judgearena.tasks.registry import get_packaged_task
 from judgearena.tasks.schema import HuggingFaceDatasetSource, ResolvedTaskSpec
@@ -57,6 +58,21 @@ def _download_source(
         if name == "examples"
         else local_tables_path
     )
+    if (
+        name == "examples"
+        and HF_HUB_OFFLINE
+        and all(
+            any(
+                path.is_file() for path in (local_dir / language).glob("test-*.parquet")
+            )
+            for language in _selected_languages(task)
+        )
+    ):
+        return
+    if name == "outputs" and task.selection is not None:
+        path = local_dir / "model_outputs" / f"{task.task}.csv.zip"
+        if path.is_file():
+            return
     snapshot_download(
         repo_id=declared_source.repo_id,
         repo_type="dataset",
@@ -87,17 +103,20 @@ def _load_source_frames(
     selected = set(_selected_languages(task))
 
     frames: list[pd.DataFrame] = []
+    found_languages: set[str] = set()
     for parquet_path in sorted(source_root.rglob("*.parquet")):
         language = parquet_path.parent.name
         if language not in selected:
             continue
         frame = pd.read_parquet(parquet_path)
         frame["lang"] = language
         frames.append(frame)
+        found_languages.add(language)
 
-    if not frames:
+    missing = selected - found_languages
+    if missing:
         raise FileNotFoundError(
-            f"No m-ArenaHard parquet files for {sorted(selected)} under {source_root}."
+            f"No m-ArenaHard parquet files for {sorted(missing)} under {source_root}."
         )
 
     df = pd.concat(frames, ignore_index=True)

judgearena/datasets/mt_bench.py

@@ -143,7 +143,10 @@ def _download_mt_bench(
 
 def download_mt_bench(local_dir: Path | None = None) -> tuple[Path, Path | None]:
     """Compatibility wrapper downloading the registered MT-Bench sources."""
-    return _download_mt_bench(_task(), local_dir or data_root / "mt-bench")
+    task = _task()
+    return _download_mt_bench(
+        task, local_dir or _task_cache_dir(task, data_root / "tables")
+    )
 
 
 def download_mt_bench_model_answer(
@@ -154,7 +157,7 @@ def download_mt_bench_model_answer(
 ) -> Path:
     """Download a cached MT-Bench model-answer file if missing."""
     resolved = task or _task()
-    root = local_dir or data_root / "mt-bench"
+    root = local_dir or _task_cache_dir(resolved, data_root / "tables")
     answer_path = root / "data" / "mt_bench" / "model_answer" / f"{model_id}.jsonl"
     if answer_path.exists():
         return answer_path
@@ -344,4 +347,5 @@ def load_task_model_outputs(
 
 def load_mt_bench() -> pd.DataFrame:
     """Compatibility wrapper loading the registered MT-Bench task."""
-    return _load_mt_bench(_task(), data_root / "mt-bench")
+    task = _task()
+    return _load_mt_bench(task, _task_cache_dir(task, data_root / "tables"))

judgearena/tasks/README.md

@@ -26,6 +26,24 @@ Executable behavior lives outside the definitions:
 - `judgearena/benchmarks/` runs the selected evaluation workflow.
 - `judgearena/prompts/` owns reusable judge prompt presets.
 
+## Task data
+
+Set `JUDGEARENA_DATA` before downloading and running a task. By default it is
+`OPENJURY_DATA` or `~/judgearena-data`. Use the same path on the download host
+and compute node, and bind it into the container.
+
+- `$JUDGEARENA_DATA/tables/`: task-owned inputs and archived outputs. MT-Bench
+  keeps its pinned questions and baseline answers under `tables/_sources/mt-bench/`.
+- `$JUDGEARENA_DATA/cache/`: generated model responses.
+- `HF_HOME`: separate Hugging Face model/cache storage.
+- `--run.result_folder`: run results, separate from task inputs.
+
+`judgearena tasks download mt-bench` prepares its sources under `tables/`.
+Use the same `JUDGEARENA_DATA` for evaluation. Do not move answers to the old
+`$JUDGEARENA_DATA/mt-bench` path; re-download into the current task path instead.
+Some task variants lack archived baseline answers and require generation from
+the declared baseline model.
+
 ## Add a task using existing components
 
 Create a public YAML file under `definitions/<family>/`. Files beginning with

judgearena/tasks/cli.py

@@ -27,6 +27,9 @@ def _parser() -> argparse.ArgumentParser:
 
     validate = commands.add_parser("validate", help="Validate packaged tasks.")
     validate.add_argument("task", nargs="?")
+
+    download = commands.add_parser("download", help="Download a packaged task.")
+    download.add_argument("task", nargs="+")
     return parser
 
 
@@ -56,6 +59,13 @@ def run_task_command(
                 print(f"Validated task {args.task!r}.")
             else:
                 print(f"Validated {len(tasks)} task(s).")
+        elif args.command == "download":
+            selected = [_require(parser, tasks, name) for name in args.task]
+            from judgearena.utils.io import data_root, download_hf
+
+            for task in selected:
+                download_hf(task.task, data_root / "tables")
+                print(f"Downloaded task {task.task!r}.")
     except TaskDefinitionError as exc:
         parser.error(str(exc))
 

judgearena/utils/io.py

@@ -8,20 +8,11 @@
 import pandas as pd
 
 from judgearena.log import get_logger
+from judgearena.paths import data_root
 
 logger = get_logger(__name__)
 
 
-def _data_root_path() -> Path:
-    raw = os.environ.get("JUDGEARENA_DATA") or os.environ.get("OPENJURY_DATA")
-    if raw:
-        return Path(raw).expanduser()
-    return Path("~/judgearena-data/").expanduser()
-
-
-data_root = _data_root_path()
-
-
 def download_hf(name: str, local_path: Path):
     """Download the HF sources a packaged task declares in its YAML."""
     from judgearena.tasks.registry import get_packaged_task
@@ -34,6 +25,9 @@ def download_hf(name: str, local_path: Path):
     resolve_download_adapter(resolved_task.spec.dataset.adapter).download(
         resolved_task, local_path
     )
+    from judgearena.benchmarks.scoring import prefetch_metrics
+
+    prefetch_metrics(resolved_task.spec.protocol.scoring.metrics)
 
 
 def read_df(filename: Path, **pandas_kwargs) -> pd.DataFrame:

pyproject.toml

@@ -27,7 +27,6 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 dependencies = [
-    "datasets>=4.0.0",
     "fast-langdetect>=1.0.0",
     "huggingface-hub>=0.34.4",
     "langchain-core>=0.3.27",
@@ -36,6 +35,7 @@ dependencies = [
     "langchain-openai>=0.3.32",
     "langchain-together>=0.3.1",
     "pandas>=2.3.2",
+    "pyarrow",
     "pyyaml>=6.0.2",
     "scikit-learn>=1.8.0,<1.10",
     "tiktoken>=0.7.0",
@@ -102,6 +102,7 @@ quote-style = "double"
 indent-style = "space"
 
 [project.optional-dependencies]
+scripts = ["datasets>=4.0.0"]
 # vLLM 0.19.1 is the pinned judge/battle runtime on this branch. Its own
 # Requires-Dist is `transformers!=5.0.*,...,!=5.5.0,>=4.56.0`, so the
 # only sub-range >=5.x that vLLM 0.19.1 accepts is >=5.5.1 — hence the

tests/test_elo_task_runtime.py

@@ -2,6 +2,7 @@
 
 import pandas as pd
 
+import judgearena.arenas_utils as arenas_utils
 import judgearena.datasets.arena_battles as arena_battles
 from judgearena.tasks.registry import get_packaged_task
 
@@ -36,6 +37,7 @@ def fake_load_arena_dataframe(arena, *, dataset_sources):
 def test_elo_dataset_download_uses_pinned_task_source(monkeypatch, tmp_path):
     task = _elo_task("elo-lmarena-100k")
     captured = {}
+    monkeypatch.setattr(arena_battles, "HF_HUB_OFFLINE", True)
     monkeypatch.setattr(
         arena_battles,
         "snapshot_download",
@@ -47,3 +49,26 @@ def test_elo_dataset_download_uses_pinned_task_source(monkeypatch, tmp_path):
     assert captured["repo_id"] == "lmarena-ai/arena-human-preference-100k"
     assert captured["revision"] == "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a"
     assert captured["allow_patterns"] == ("data/*.parquet",)
+    assert captured["local_files_only"] is True
+
+
+def test_elo_runtime_uses_cached_snapshot_offline(monkeypatch):
+    task = _elo_task("elo-lmarena-100k")
+    repo_id = "lmarena-ai/arena-human-preference-100k"
+    captured = {}
+    monkeypatch.setattr(arenas_utils, "HF_HUB_OFFLINE", True)
+
+    def cached_snapshot(**kwargs):
+        captured.update(kwargs)
+        return "/cached/snapshot"
+
+    monkeypatch.setattr(arenas_utils, "snapshot_download", cached_snapshot)
+    result = arenas_utils._download_arena_dataset(
+        repo_id=repo_id,
+        default_allow_patterns="data/*.parquet",
+        dataset_sources={repo_id: task.spec.dataset.sources["lmarena_100k"]},
+    )
+
+    assert result == "/cached/snapshot"
+    assert captured["revision"] == "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a"
+    assert captured["local_files_only"] is True