From 932df22317e15885d17198562b18d84bf3e8c412 Mon Sep 17 00:00:00 2001
From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com>
Date: Fri, 9 Oct 2026 09:45:01 +0200
Subject: [PATCH 1/4] Resolve OpenRouter reference prices and freeze rates per
 tuning session

---
 judgearena/tuning/pricing.py |  96 ++++++++++++++++++++++++++++---
 judgearena/tuning/runner.py  |  30 ++++++----
 tests/test_tuning.py         |  20 +++++--
 tests/test_tuning_pricing.py | 108 +++++++++++++++++++++++++++++++++++
 4 files changed, 227 insertions(+), 27 deletions(-)
 create mode 100644 tests/test_tuning_pricing.py

diff --git a/judgearena/tuning/pricing.py b/judgearena/tuning/pricing.py
index be671c95..7acff2e4 100644
--- a/judgearena/tuning/pricing.py
+++ b/judgearena/tuning/pricing.py
@@ -1,6 +1,12 @@
 """Explicit judge-token prices used by the tuning cost objective."""
 
+import json
 from dataclasses import dataclass
+from pathlib import Path
+from urllib.error import URLError
+from urllib.request import urlopen
+
+_OPENROUTER_URL = "https://openrouter.ai/api/v1/models"
 
 
 @dataclass(frozen=True)
@@ -10,19 +16,91 @@ class TokenPrice:
     source: str
 
 
-def resolve_prices(models, overrides, *, require_cost=True) -> dict[str, TokenPrice]:
-    """Resolve explicit USD-per-million-token rates before searching."""
+def _fetch_openrouter(catalog_cache: Path | None) -> list[dict]:
+    try:
+        with urlopen(_OPENROUTER_URL, timeout=10) as response:
+            records = json.load(response)["data"]
+        if catalog_cache is not None:
+            catalog_cache.parent.mkdir(parents=True, exist_ok=True)
+            catalog_cache.write_text(json.dumps(records), encoding="utf-8")
+        return records
+    except (URLError, TimeoutError, OSError) as error:
+        if catalog_cache is not None and catalog_cache.exists():
+            cached = json.loads(catalog_cache.read_text(encoding="utf-8"))
+            if isinstance(cached, dict):
+                return [
+                    {
+                        "id": model_id,
+                        "pricing": {"prompt": price[0], "completion": price[1]},
+                    }
+                    for model_id, price in cached.items()
+                ]
+            if isinstance(cached, list):
+                return cached
+        raise ValueError(
+            "Could not fetch OpenRouter pricing and no cached catalog is available; "
+            "fetch the catalog on a login node or set "
+            "tune_judge.price_per_million_tokens"
+        ) from error
+
+
+def _find_price(model: str, records: list[dict]) -> TokenPrice | None:
+    requested_id = model.split("/", 1)[-1].lower()
+    suffix = "/".join(requested_id.split("/")[-2:])
+    record = next(
+        (
+            item
+            for item in records
+            if str(item.get("hugging_face_id") or "").lower() == requested_id
+        ),
+        None,
+    )
+    if record is None:
+        record = next(
+            (item for item in records if str(item.get("id") or "").lower() == suffix),
+            None,
+        )
+    if record is None:
+        return None
+    rates = record.get("pricing", {})
+    if "prompt" not in rates or "completion" not in rates:
+        return None
+    return TokenPrice(
+        float(rates["prompt"]) * 1_000_000,
+        float(rates["completion"]) * 1_000_000,
+        "openrouter_reference",
+    )
+
+
+def resolve_prices(
+    models,
+    overrides,
+    *,
+    catalog_cache: Path | None = None,
+    require_cost=True,
+) -> dict[str, TokenPrice]:
+    """Resolve explicit or OpenRouter reference USD-per-million-token rates."""
     prices = {}
+    unresolved = []
     for model in sorted(models):
-        if model not in overrides:
-            continue
-        rate = overrides[model]
-        if isinstance(rate, (int, float)):
-            rate = {"input": rate, "output": rate}
-        prices[model] = TokenPrice(rate["input"], rate["output"], "user_override")
+        rate = overrides.get(model)
+        if rate is not None:
+            if isinstance(rate, (int, float)):
+                rate = {"input": rate, "output": rate}
+            prices[model] = TokenPrice(
+                float(rate["input"]), float(rate["output"]), "user_override"
+            )
+        else:
+            unresolved.append(model)
+    records = _fetch_openrouter(catalog_cache) if unresolved else []
+    for model in unresolved:
+        price = _find_price(model, records)
+        if price is not None:
+            prices[model] = price
     missing = sorted(set(models) - prices.keys())
     if missing and require_cost:
         raise ValueError(
-            f"No token price for {missing}; set tune_judge.price_per_million_tokens"
+            f"No token price for {missing}; set tune_judge.price_per_million_tokens "
+            f"overrides for: {missing}"
         )
     return prices
diff --git a/judgearena/tuning/runner.py b/judgearena/tuning/runner.py
index d6e59565..d9c477ac 100644
--- a/judgearena/tuning/runner.py
+++ b/judgearena/tuning/runner.py
@@ -253,21 +253,27 @@ def run_tune_judge(
             models.add(trial_cfg.judge.model)
     if "judge.model" not in specs:
         models.add(cfg.judge.model)
-    prices = resolve_prices(
-        models,
-        tuning.price_per_million_tokens,
-        require_cost="cost" in tuning.objectives,
-    )
-    runner = _TrialRunner(runner.base, prices, execute_trial)
     tune_dir = prepare_session(cfg)
     price_path = tune_dir / "prices.json"
-    snapshot = {model: asdict(price) for model, price in prices.items()}
     if price_path.exists():
-        stored = json.loads(price_path.read_text())
-        if stored != snapshot:
-            raise ValueError("Resolved prices changed for this tuning session")
-    elif not tuning.search_only:
-        price_path.write_text(json.dumps(snapshot, indent=2))
+        prices = {
+            model: TokenPrice(**price)
+            for model, price in json.loads(price_path.read_text()).items()
+        }
+    else:
+        prices = resolve_prices(
+            models,
+            tuning.price_per_million_tokens,
+            catalog_cache=Path(cfg.run.store_root or cfg.run.result_folder)
+            / "openrouter_pricing.json",
+            require_cost="cost" in tuning.objectives,
+        )
+        price_path.write_text(
+            json.dumps(
+                {model: asdict(price) for model, price in prices.items()}, indent=2
+            )
+        )
+    runner = _TrialRunner(runner.base, prices, execute_trial)
     logger.info("Tuning %s in %s", cfg.task, tune_dir)
     _search(runner, cfg, tune_dir, space)
     if tuning.search_only:
diff --git a/tests/test_tuning.py b/tests/test_tuning.py
index dff53083..05747201 100644
--- a/tests/test_tuning.py
+++ b/tests/test_tuning.py
@@ -98,7 +98,7 @@ def _fake_trial(config_path):
 @pytest.mark.parametrize(
     "algorithm", ["neps_priorband", "neps_hyperband", "mo_hyperband", "primo"]
 )
-def test_native_optimizer_smoke(tmp_path, token_counter, algorithm):
+def test_native_optimizer_smoke(tmp_path, token_counter, algorithm, monkeypatch):
     pytest.importorskip("neps")
     cfg = _tuning_config(tmp_path, algorithm)
     calls = []
@@ -127,11 +127,16 @@ def execute(path):
     assert load_config(calls[-1]).meta_eval.battles_per_model == 3
     before = len(calls)
     (folder / "test_results.parquet").unlink()
-    resumed = runner.run_tune_judge(cfg, task, execute_trial=execute)
-    pd.testing.assert_frame_equal(results, resumed)
-    assert len(calls) == before  # Completed search and held-out artifacts are reused.
     helper = _tuning_config(tmp_path, search_only=True)
-    assert runner.run_tune_judge(helper, task, execute_trial=execute).empty
+    with monkeypatch.context() as patch:
+        patch.setattr(
+            runner,
+            "resolve_prices",
+            lambda *a, **kw: pytest.fail("Refetched session prices"),
+        )
+        resumed = runner.run_tune_judge(cfg, task, execute_trial=execute)
+        assert runner.run_tune_judge(helper, task, execute_trial=execute).empty
+    pd.testing.assert_frame_equal(results, resumed)
     assert len(calls) == before
     incompatible = _tuning_config(
         tmp_path, price_per_million_tokens={"Dummy/judge": 2.0, "Dummy/other": 2.0}
@@ -199,7 +204,10 @@ def test_cost_counts_both_cached_orientations(tmp_path, token_counter):
     assert "cost" not in objective
 
 
-def test_prices_required_before_execution(tmp_path):
+def test_prices_required_before_execution(tmp_path, monkeypatch):
+    from judgearena.tuning import pricing
+
+    monkeypatch.setattr(pricing, "_fetch_openrouter", lambda _: [])
     pytest.importorskip("neps")
     cfg = _tuning_config(tmp_path, "mo_hyperband", price_per_million_tokens={})
     with pytest.raises(ValueError, match="No token price"):
diff --git a/tests/test_tuning_pricing.py b/tests/test_tuning_pricing.py
new file mode 100644
index 00000000..18351a44
--- /dev/null
+++ b/tests/test_tuning_pricing.py
@@ -0,0 +1,108 @@
+import json
+from urllib.error import URLError
+
+import pytest
+
+from judgearena.tuning import pricing
+from judgearena.tuning.pricing import TokenPrice, resolve_prices
+
+
+def test_overrides_do_not_fetch_catalog(monkeypatch):
+    def fail(*args, **kwargs):
+        raise AssertionError("all prices were overridden")
+
+    monkeypatch.setattr(pricing, "urlopen", fail)
+    result = resolve_prices(
+        ["VLLM/local", "OpenRouter/hosted"],
+        {"VLLM/local": 1.2, "OpenRouter/hosted": {"input": 0.2, "output": 0.7}},
+    )
+    assert result["VLLM/local"] == TokenPrice(1.2, 1.2, "user_override")
+    assert result["OpenRouter/hosted"] == TokenPrice(0.2, 0.7, "user_override")
+
+
+def test_huggingface_id_match_precedes_suffix_for_every_provider(tmp_path, monkeypatch):
+    records = [
+        {
+            "id": "org/model",
+            "pricing": {"prompt": "0.000001", "completion": "0.000002"},
+        },
+        {
+            "id": "elsewhere/different",
+            "hugging_face_id": "ORG/Model",
+            "pricing": {"prompt": "0.000003", "completion": "0.000004"},
+        },
+    ]
+    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _Response(records))
+    cache = tmp_path / "openrouter_pricing.json"
+    for model in ("VLLM/org/model", "Provider/org/model"):
+        result = resolve_prices([model], {}, catalog_cache=cache)
+        assert result[model] == TokenPrice(3, 4, "openrouter_reference")
+        assert json.loads(cache.read_text()) == records
+
+
+def test_suffix_match(monkeypatch):
+    monkeypatch.setattr(
+        pricing,
+        "urlopen",
+        lambda *a, **k: _Response(
+            [
+                {
+                    "id": "org/model",
+                    "pricing": {"prompt": "0.000005", "completion": "0.000009"},
+                }
+            ]
+        ),
+    )
+    assert resolve_prices(["Hosted/org/model"], {})["Hosted/org/model"] == TokenPrice(
+        5, 9, "openrouter_reference"
+    )
+
+
+def test_fetch_failure_uses_cached_catalog(tmp_path, monkeypatch):
+    cache = tmp_path / "openrouter_pricing.json"
+    cache.write_text(
+        json.dumps(
+            [
+                {
+                    "id": "org/model",
+                    "pricing": {"prompt": "0.000002", "completion": "0.000006"},
+                }
+            ]
+        )
+    )
+    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _raise_url_error())
+    price = resolve_prices(["Backend/org/model"], {}, catalog_cache=cache)
+    assert price["Backend/org/model"] == TokenPrice(2, 6, "openrouter_reference")
+
+
+def test_fetch_failure_without_cache_has_actionable_error(tmp_path, monkeypatch):
+    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _raise_url_error())
+    with pytest.raises(ValueError, match="login node.*price_per_million_tokens"):
+        resolve_prices(
+            ["Backend/org/model"], {}, catalog_cache=tmp_path / "catalog.json"
+        )
+
+
+def test_unpriced_model_fails_for_cost_objective(monkeypatch):
+    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _Response([]))
+    with pytest.raises(ValueError, match="unknown.*overrides"):
+        resolve_prices(["VLLM/unknown"], {})
+    assert resolve_prices(["VLLM/unknown"], {}, require_cost=False) == {}
+
+
+class _Response:
+    def __init__(self, records):
+        self.records = records
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return None
+
+    def read(self):
+        return json.dumps({"data": self.records}).encode()
+
+
+def _raise_url_error():
+    raise URLError("offline")

From b8480ed102ac1b43983b563cda7417868c800797 Mon Sep 17 00:00:00 2001
From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com>
Date: Fri, 9 Oct 2026 10:57:16 +0200
Subject: [PATCH 2/4] Centralize OpenRouter reference pricing and native token
 cost calculation

---
 judgearena/{tuning => }/pricing.py | 42 +++++++++++++++++++++---------
 judgearena/tuning/runner.py        |  2 +-
 tests/test_tuning_pricing.py       | 40 +++++++++++++++++++++++++---
 3 files changed, 68 insertions(+), 16 deletions(-)
 rename judgearena/{tuning => }/pricing.py (72%)

diff --git a/judgearena/tuning/pricing.py b/judgearena/pricing.py
similarity index 72%
rename from judgearena/tuning/pricing.py
rename to judgearena/pricing.py
index 7acff2e4..3b465289 100644
--- a/judgearena/tuning/pricing.py
+++ b/judgearena/pricing.py
@@ -1,4 +1,4 @@
-"""Explicit judge-token prices used by the tuning cost objective."""
+"""Shared reference token pricing and request cost calculation."""
 
 import json
 from dataclasses import dataclass
@@ -6,6 +6,8 @@
 from urllib.error import URLError
 from urllib.request import urlopen
 
+from judgearena.usage import RequestUsage
+
 _OPENROUTER_URL = "https://openrouter.ai/api/v1/models"
 
 
@@ -16,6 +18,24 @@ class TokenPrice:
     source: str
 
 
+def reference_cost(usage: RequestUsage, price: TokenPrice) -> float:
+    """Calculate USD from native input/output token counts and per-million rates."""
+    missing = [
+        name
+        for name in ("input_tokens", "output_tokens")
+        if getattr(usage, name) is None
+    ]
+    if missing:
+        fields = " and ".join(missing)
+        raise ValueError(
+            f"Cannot calculate reference cost: missing native {fields}; "
+            "provide provider-reported input_tokens and output_tokens usage"
+        )
+    return (
+        usage.input_tokens * price.input + usage.output_tokens * price.output
+    ) / 1_000_000
+
+
 def _fetch_openrouter(catalog_cache: Path | None) -> list[dict]:
     try:
         with urlopen(_OPENROUTER_URL, timeout=10) as response:
@@ -27,20 +47,16 @@ def _fetch_openrouter(catalog_cache: Path | None) -> list[dict]:
     except (URLError, TimeoutError, OSError) as error:
         if catalog_cache is not None and catalog_cache.exists():
             cached = json.loads(catalog_cache.read_text(encoding="utf-8"))
-            if isinstance(cached, dict):
-                return [
-                    {
-                        "id": model_id,
-                        "pricing": {"prompt": price[0], "completion": price[1]},
-                    }
-                    for model_id, price in cached.items()
-                ]
             if isinstance(cached, list):
                 return cached
+            raise ValueError(
+                "Cached OpenRouter pricing catalog uses an unsupported format; "
+                "refresh the catalog or set price_per_million_tokens overrides"
+            ) from error
         raise ValueError(
             "Could not fetch OpenRouter pricing and no cached catalog is available; "
             "fetch the catalog on a login node or set "
-            "tune_judge.price_per_million_tokens"
+            "price_per_million_tokens overrides"
         ) from error
 
 
@@ -80,6 +96,8 @@ def resolve_prices(
     require_cost=True,
 ) -> dict[str, TokenPrice]:
     """Resolve explicit or OpenRouter reference USD-per-million-token rates."""
+    if not require_cost:
+        return {}
     prices = {}
     unresolved = []
     for model in sorted(models):
@@ -98,9 +116,9 @@ def resolve_prices(
         if price is not None:
             prices[model] = price
     missing = sorted(set(models) - prices.keys())
-    if missing and require_cost:
+    if missing:
         raise ValueError(
-            f"No token price for {missing}; set tune_judge.price_per_million_tokens "
+            f"No token price for {missing}; set price_per_million_tokens "
             f"overrides for: {missing}"
         )
     return prices
diff --git a/judgearena/tuning/runner.py b/judgearena/tuning/runner.py
index d9c477ac..49b80804 100644
--- a/judgearena/tuning/runner.py
+++ b/judgearena/tuning/runner.py
@@ -30,7 +30,7 @@
 if TYPE_CHECKING:
     from judgearena.tasks.schema import ResolvedTaskSpec
 
-from judgearena.tuning.pricing import TokenPrice, resolve_prices
+from judgearena.pricing import TokenPrice, resolve_prices
 from judgearena.tuning.session import collect_trials, prepare_session
 
 logger = get_logger(__name__)
diff --git a/tests/test_tuning_pricing.py b/tests/test_tuning_pricing.py
index 18351a44..c13071ad 100644
--- a/tests/test_tuning_pricing.py
+++ b/tests/test_tuning_pricing.py
@@ -3,8 +3,9 @@
 
 import pytest
 
-from judgearena.tuning import pricing
-from judgearena.tuning.pricing import TokenPrice, resolve_prices
+from judgearena import pricing
+from judgearena.pricing import TokenPrice, reference_cost, resolve_prices
+from judgearena.usage import RequestUsage
 
 
 def test_overrides_do_not_fetch_catalog(monkeypatch):
@@ -84,10 +85,43 @@ def test_fetch_failure_without_cache_has_actionable_error(tmp_path, monkeypatch)
 
 
 def test_unpriced_model_fails_for_cost_objective(monkeypatch):
+    def unexpected_fetch(*args, **kwargs):
+        raise AssertionError("agreement-only pricing must not fetch the catalog")
+
+    monkeypatch.setattr(pricing, "urlopen", unexpected_fetch)
+    assert resolve_prices(["VLLM/unknown"], {}, require_cost=False) == {}
+
     monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _Response([]))
     with pytest.raises(ValueError, match="unknown.*overrides"):
         resolve_prices(["VLLM/unknown"], {})
-    assert resolve_prices(["VLLM/unknown"], {}, require_cost=False) == {}
+
+
+def test_reference_cost_requires_native_counts_and_uses_reported_zero():
+    with pytest.raises(
+        ValueError, match="missing native input_tokens and output_tokens"
+    ):
+        reference_cost(
+            RequestUsage(stage="judging", reasoning_tokens=12),
+            TokenPrice(1, 2, "ref"),
+        )
+
+    usage = RequestUsage(
+        stage="judging", input_tokens=0, output_tokens=3, reasoning_tokens=100
+    )
+    assert reference_cost(usage, TokenPrice(1_000_000, 2_000_000, "ref")) == 6
+
+    with pytest.raises(ValueError, match="missing native output_tokens"):
+        reference_cost(
+            RequestUsage(stage="judging", input_tokens=3), TokenPrice(1, 2, "ref")
+        )
+
+
+def test_fetch_failure_rejects_legacy_dictionary_catalog(tmp_path, monkeypatch):
+    cache = tmp_path / "catalog.json"
+    cache.write_text(json.dumps({"org/model": [1, 2]}))
+    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _raise_url_error())
+    with pytest.raises(ValueError, match="unsupported format.*refresh.*overrides"):
+        resolve_prices(["Backend/org/model"], {}, catalog_cache=cache)
 
 
 class _Response:

From d9f0a39c4b7614368e9ce6ad790436e3335fc7f0 Mon Sep 17 00:00:00 2001
From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com>
Date: Fri, 9 Oct 2026 16:50:29 +0200
Subject: [PATCH 3/4] Share pricing fixtures and parameterize catalog
 resolution cases

---
 tests/test_tuning_pricing.py | 167 ++++++++++++++++-------------------
 1 file changed, 77 insertions(+), 90 deletions(-)

diff --git a/tests/test_tuning_pricing.py b/tests/test_tuning_pricing.py
index c13071ad..836449d1 100644
--- a/tests/test_tuning_pricing.py
+++ b/tests/test_tuning_pricing.py
@@ -1,4 +1,6 @@
 import json
+from contextlib import nullcontext
+from io import BytesIO
 from urllib.error import URLError
 
 import pytest
@@ -8,6 +10,21 @@
 from judgearena.usage import RequestUsage
 
 
+@pytest.fixture
+def catalog():
+    return [
+        {
+            "id": "org/model",
+            "pricing": {"prompt": "0.000001", "completion": "0.000002"},
+        },
+        {
+            "id": "elsewhere/different",
+            "hugging_face_id": "ORG/Model",
+            "pricing": {"prompt": "0.000003", "completion": "0.000004"},
+        },
+    ]
+
+
 def test_overrides_do_not_fetch_catalog(monkeypatch):
     def fail(*args, **kwargs):
         raise AssertionError("all prices were overridden")
@@ -21,67 +38,53 @@ def fail(*args, **kwargs):
     assert result["OpenRouter/hosted"] == TokenPrice(0.2, 0.7, "user_override")
 
 
-def test_huggingface_id_match_precedes_suffix_for_every_provider(tmp_path, monkeypatch):
-    records = [
-        {
-            "id": "org/model",
-            "pricing": {"prompt": "0.000001", "completion": "0.000002"},
-        },
-        {
-            "id": "elsewhere/different",
-            "hugging_face_id": "ORG/Model",
-            "pricing": {"prompt": "0.000003", "completion": "0.000004"},
-        },
-    ]
-    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _Response(records))
-    cache = tmp_path / "openrouter_pricing.json"
-    for model in ("VLLM/org/model", "Provider/org/model"):
-        result = resolve_prices([model], {}, catalog_cache=cache)
-        assert result[model] == TokenPrice(3, 4, "openrouter_reference")
-        assert json.loads(cache.read_text()) == records
-
-
-def test_suffix_match(monkeypatch):
+@pytest.mark.parametrize(
+    ("model", "hf_match", "expected"),
+    [
+        ("VLLM/org/model", True, TokenPrice(3, 4, "openrouter_reference")),
+        ("Hosted/org/model", True, TokenPrice(3, 4, "openrouter_reference")),
+        ("Gateway/org/model", True, TokenPrice(3, 4, "openrouter_reference")),
+        ("VLLM/org/model", False, TokenPrice(1, 2, "openrouter_reference")),
+    ],
+)
+def test_reference_model_matching(
+    model, hf_match, expected, catalog, monkeypatch, tmp_path
+):
+    records = catalog if hf_match else catalog[:1]
     monkeypatch.setattr(
         pricing,
         "urlopen",
-        lambda *a, **k: _Response(
-            [
-                {
-                    "id": "org/model",
-                    "pricing": {"prompt": "0.000005", "completion": "0.000009"},
-                }
-            ]
-        ),
-    )
-    assert resolve_prices(["Hosted/org/model"], {})["Hosted/org/model"] == TokenPrice(
-        5, 9, "openrouter_reference"
+        lambda *a, **k: nullcontext(BytesIO(json.dumps({"data": records}).encode())),
     )
-
-
-def test_fetch_failure_uses_cached_catalog(tmp_path, monkeypatch):
-    cache = tmp_path / "openrouter_pricing.json"
-    cache.write_text(
-        json.dumps(
-            [
-                {
-                    "id": "org/model",
-                    "pricing": {"prompt": "0.000002", "completion": "0.000006"},
-                }
-            ]
+    cache = tmp_path / "catalog.json"
+    assert resolve_prices([model], {}, catalog_cache=cache) == {model: expected}
+    assert json.loads(cache.read_text()) == records
+
+
+@pytest.mark.parametrize(
+    "cache_content,error",
+    [
+        ("catalog", None),
+        (None, "login node.*price_per_million_tokens"),
+        ({"org/model": [1, 2]}, "unsupported format.*refresh.*overrides"),
+    ],
+)
+def test_offline_catalog_resolution(
+    tmp_path, monkeypatch, catalog, cache_content, error
+):
+    cache = tmp_path / "catalog.json"
+    if cache_content is not None:
+        cache.write_text(
+            json.dumps(catalog if cache_content == "catalog" else cache_content)
         )
-    )
     monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _raise_url_error())
-    price = resolve_prices(["Backend/org/model"], {}, catalog_cache=cache)
-    assert price["Backend/org/model"] == TokenPrice(2, 6, "openrouter_reference")
-
-
-def test_fetch_failure_without_cache_has_actionable_error(tmp_path, monkeypatch):
-    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _raise_url_error())
-    with pytest.raises(ValueError, match="login node.*price_per_million_tokens"):
-        resolve_prices(
-            ["Backend/org/model"], {}, catalog_cache=tmp_path / "catalog.json"
-        )
+    if error:
+        with pytest.raises(ValueError, match=error):
+            resolve_prices(["VLLM/org/model"], {}, catalog_cache=cache)
+    else:
+        assert resolve_prices(["VLLM/org/model"], {}, catalog_cache=cache) == {
+            "VLLM/org/model": TokenPrice(3, 4, "openrouter_reference")
+        }
 
 
 def test_unpriced_model_fails_for_cost_objective(monkeypatch):
@@ -91,52 +94,36 @@ def unexpected_fetch(*args, **kwargs):
     monkeypatch.setattr(pricing, "urlopen", unexpected_fetch)
     assert resolve_prices(["VLLM/unknown"], {}, require_cost=False) == {}
 
-    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _Response([]))
+    monkeypatch.setattr(
+        pricing,
+        "urlopen",
+        lambda *a, **k: nullcontext(BytesIO(b'{"data": []}')),
+    )
     with pytest.raises(ValueError, match="unknown.*overrides"):
         resolve_prices(["VLLM/unknown"], {})
 
 
-def test_reference_cost_requires_native_counts_and_uses_reported_zero():
-    with pytest.raises(
-        ValueError, match="missing native input_tokens and output_tokens"
-    ):
-        reference_cost(
+@pytest.mark.parametrize(
+    ("usage", "missing"),
+    [
+        (
             RequestUsage(stage="judging", reasoning_tokens=12),
-            TokenPrice(1, 2, "ref"),
-        )
+            "input_tokens and output_tokens",
+        ),
+        (RequestUsage(stage="judging", input_tokens=3), "output_tokens"),
+    ],
+)
+def test_reference_cost_requires_native_counts(usage, missing):
+    with pytest.raises(ValueError, match=f"missing native {missing}"):
+        reference_cost(usage, TokenPrice(1, 2, "ref"))
+
 
+def test_reference_cost_uses_native_counts_including_zero():
     usage = RequestUsage(
         stage="judging", input_tokens=0, output_tokens=3, reasoning_tokens=100
     )
     assert reference_cost(usage, TokenPrice(1_000_000, 2_000_000, "ref")) == 6
 
-    with pytest.raises(ValueError, match="missing native output_tokens"):
-        reference_cost(
-            RequestUsage(stage="judging", input_tokens=3), TokenPrice(1, 2, "ref")
-        )
-
-
-def test_fetch_failure_rejects_legacy_dictionary_catalog(tmp_path, monkeypatch):
-    cache = tmp_path / "catalog.json"
-    cache.write_text(json.dumps({"org/model": [1, 2]}))
-    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _raise_url_error())
-    with pytest.raises(ValueError, match="unsupported format.*refresh.*overrides"):
-        resolve_prices(["Backend/org/model"], {}, catalog_cache=cache)
-
-
-class _Response:
-    def __init__(self, records):
-        self.records = records
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        return None
-
-    def read(self):
-        return json.dumps({"data": self.records}).encode()
-
 
 def _raise_url_error():
     raise URLError("offline")

From 3a498b867fa54ae46b2f3ee1735192aba73c64dd Mon Sep 17 00:00:00 2001
From: Erlis Lushtaku <59629249+ErlisLushtaku@users.noreply.github.com>
Date: Fri, 9 Oct 2026 17:10:11 +0200
Subject: [PATCH 4/4] Keep concise reference pricing contract tests

---
 tests/test_tuning_pricing.py | 145 ++++++++++-------------------------
 1 file changed, 41 insertions(+), 104 deletions(-)

diff --git a/tests/test_tuning_pricing.py b/tests/test_tuning_pricing.py
index 836449d1..8600f02c 100644
--- a/tests/test_tuning_pricing.py
+++ b/tests/test_tuning_pricing.py
@@ -1,7 +1,6 @@
 import json
 from contextlib import nullcontext
 from io import BytesIO
-from urllib.error import URLError
 
 import pytest
 
@@ -10,120 +9,58 @@
 from judgearena.usage import RequestUsage
 
 
-@pytest.fixture
-def catalog():
-    return [
-        {
-            "id": "org/model",
-            "pricing": {"prompt": "0.000001", "completion": "0.000002"},
-        },
-        {
-            "id": "elsewhere/different",
-            "hugging_face_id": "ORG/Model",
-            "pricing": {"prompt": "0.000003", "completion": "0.000004"},
-        },
-    ]
-
-
-def test_overrides_do_not_fetch_catalog(monkeypatch):
-    def fail(*args, **kwargs):
-        raise AssertionError("all prices were overridden")
-
-    monkeypatch.setattr(pricing, "urlopen", fail)
-    result = resolve_prices(
-        ["VLLM/local", "OpenRouter/hosted"],
-        {"VLLM/local": 1.2, "OpenRouter/hosted": {"input": 0.2, "output": 0.7}},
-    )
-    assert result["VLLM/local"] == TokenPrice(1.2, 1.2, "user_override")
-    assert result["OpenRouter/hosted"] == TokenPrice(0.2, 0.7, "user_override")
-
-
-@pytest.mark.parametrize(
-    ("model", "hf_match", "expected"),
-    [
-        ("VLLM/org/model", True, TokenPrice(3, 4, "openrouter_reference")),
-        ("Hosted/org/model", True, TokenPrice(3, 4, "openrouter_reference")),
-        ("Gateway/org/model", True, TokenPrice(3, 4, "openrouter_reference")),
-        ("VLLM/org/model", False, TokenPrice(1, 2, "openrouter_reference")),
-    ],
-)
-def test_reference_model_matching(
-    model, hf_match, expected, catalog, monkeypatch, tmp_path
-):
-    records = catalog if hf_match else catalog[:1]
-    monkeypatch.setattr(
-        pricing,
-        "urlopen",
-        lambda *a, **k: nullcontext(BytesIO(json.dumps({"data": records}).encode())),
-    )
-    cache = tmp_path / "catalog.json"
-    assert resolve_prices([model], {}, catalog_cache=cache) == {model: expected}
-    assert json.loads(cache.read_text()) == records
+def test_override_and_agreement_only_skip_catalog(monkeypatch):
+    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: pytest.fail("fetched"))
+    assert resolve_prices(["VLLM/local"], {"VLLM/local": 1.2}) == {
+        "VLLM/local": TokenPrice(1.2, 1.2, "user_override")
+    }
+    assert resolve_prices(["VLLM/local"], {}, require_cost=False) == {}
 
 
 @pytest.mark.parametrize(
-    "cache_content,error",
-    [
-        ("catalog", None),
-        (None, "login node.*price_per_million_tokens"),
-        ({"org/model": [1, 2]}, "unsupported format.*refresh.*overrides"),
-    ],
+    "provider,hf_match", [("VLLM", True), ("OpenRouter", True), ("VLLM", False)]
 )
-def test_offline_catalog_resolution(
-    tmp_path, monkeypatch, catalog, cache_content, error
+def test_reference_matching_and_catalog_snapshot(
+    provider, hf_match, monkeypatch, tmp_path
 ):
-    cache = tmp_path / "catalog.json"
-    if cache_content is not None:
-        cache.write_text(
-            json.dumps(catalog if cache_content == "catalog" else cache_content)
+    records = [dict(id="org/model", pricing=dict(prompt="1e-6", completion="2e-6"))]
+    if hf_match:
+        records.append(
+            dict(
+                id="other/model",
+                hugging_face_id="ORG/Model",
+                pricing=dict(prompt="3e-6", completion="4e-6"),
+            )
         )
-    monkeypatch.setattr(pricing, "urlopen", lambda *a, **k: _raise_url_error())
-    if error:
-        with pytest.raises(ValueError, match=error):
-            resolve_prices(["VLLM/org/model"], {}, catalog_cache=cache)
-    else:
-        assert resolve_prices(["VLLM/org/model"], {}, catalog_cache=cache) == {
-            "VLLM/org/model": TokenPrice(3, 4, "openrouter_reference")
-        }
-
-
-def test_unpriced_model_fails_for_cost_objective(monkeypatch):
-    def unexpected_fetch(*args, **kwargs):
-        raise AssertionError("agreement-only pricing must not fetch the catalog")
-
-    monkeypatch.setattr(pricing, "urlopen", unexpected_fetch)
-    assert resolve_prices(["VLLM/unknown"], {}, require_cost=False) == {}
-
+    response = json.dumps({"data": records}).encode()
     monkeypatch.setattr(
-        pricing,
-        "urlopen",
-        lambda *a, **k: nullcontext(BytesIO(b'{"data": []}')),
+        pricing, "urlopen", lambda *a, **k: nullcontext(BytesIO(response))
+    )
+    model, cache = f"{provider}/org/model", tmp_path / "prices.json"
+    expected = (
+        TokenPrice(3, 4, "openrouter_reference")
+        if hf_match
+        else TokenPrice(1, 2, "openrouter_reference")
     )
-    with pytest.raises(ValueError, match="unknown.*overrides"):
-        resolve_prices(["VLLM/unknown"], {})
+    assert resolve_prices([model], {}, catalog_cache=cache)[model] == expected
+    assert json.loads(cache.read_text()) == records
+    with pytest.raises(ValueError, match="overrides"):
+        resolve_prices(["VLLM/unknown"], {}, catalog_cache=cache)
 
 
 @pytest.mark.parametrize(
-    ("usage", "missing"),
-    [
-        (
-            RequestUsage(stage="judging", reasoning_tokens=12),
-            "input_tokens and output_tokens",
-        ),
-        (RequestUsage(stage="judging", input_tokens=3), "output_tokens"),
-    ],
+    "tokens,expected", [((0, 3), 6), ((2, 1), 4), ((None, 3), None)]
 )
-def test_reference_cost_requires_native_counts(usage, missing):
-    with pytest.raises(ValueError, match=f"missing native {missing}"):
-        reference_cost(usage, TokenPrice(1, 2, "ref"))
-
-
-def test_reference_cost_uses_native_counts_including_zero():
+def test_native_counts_and_no_double_counted_reasoning(tokens, expected):
     usage = RequestUsage(
-        stage="judging", input_tokens=0, output_tokens=3, reasoning_tokens=100
+        stage="judging",
+        input_tokens=tokens[0],
+        output_tokens=tokens[1],
+        reasoning_tokens=100,
     )
-    assert reference_cost(usage, TokenPrice(1_000_000, 2_000_000, "ref")) == 6
-
-
-def _raise_url_error():
-    raise URLError("offline")
+    price = TokenPrice(1_000_000, 2_000_000, "ref")
+    if expected is None:
+        with pytest.raises(ValueError, match="missing native"):
+            reference_cost(usage, price)
+    else:
+        assert reference_cost(usage, price) == expected