diff --git a/docs/pipelines/README.md b/docs/pipelines/README.md index b7f7105..b483e74 100644 --- a/docs/pipelines/README.md +++ b/docs/pipelines/README.md @@ -147,7 +147,7 @@ default/__/ | `file_targeting` | 0.12 | F1 over the changed-file sets (not Jaccard — F1 properly credits TP) | | `region_overlap` | 0.20 | Predicted hunks overlap oracle hunks (5-line slack) | | `similarity` | 0.10 | `SequenceMatcher` over `+`/`-` lines only (no free credit for context) | -| `llm_judge` | 0.50 | Haiku 4.5 rates semantic correctness; graceful degradation on missing API key | +| `llm_judge` | 0.50 | An LLM rates semantic correctness — Haiku 4.5 by default, or a self-hosted model via `R2E_JUDGE_ENDPOINT`; graceful degradation on missing API key | Plus a **catastrophic-size hard cap**: clamps reward to ≤ 0.40 when `size_sanity < 0.10`, so a charitable judge can't inflate scores on patches that are wildly the wrong size. @@ -187,8 +187,12 @@ harbor run -p /tmp/pr-diff-click -a claude-code \ # -a qwen-coder -m qwen/qwen3-coder # -a copilot-cli (uses GH_TOKEN) # -a mini-swe-agent · swe-agent · cursor-cli · kimi-cli · goose · ... -# The verifier's LLM-judge always uses Anthropic (Haiku) — pass -# ANTHROPIC_API_KEY via `--ve` regardless of which agent you run. +# The verifier's LLM-judge uses Anthropic (Haiku) by default — pass +# ANTHROPIC_API_KEY via `--ve` regardless of which agent you run — or +# route it to a self-hosted model with +# `--ve R2E_JUDGE_ENDPOINT=http://host.docker.internal:8000/v1 --ve R2E_JUDGE_MODEL=` +# (host.docker.internal is Docker Desktop only; on a bare Linux daemon use +# the host's LAN IP, e.g. http://$(hostname -I | cut -d' ' -f1):8000/v1). # Publish repo2rlenv push /tmp/pr-diff-click / diff --git a/docs/pipelines/pr_diff.md b/docs/pipelines/pr_diff.md index 7ab5cd0..b493681 100644 --- a/docs/pipelines/pr_diff.md +++ b/docs/pipelines/pr_diff.md @@ -52,7 +52,7 @@ The verifier captures the agent's edits as a unified diff against `base_commit`, | `file_targeting` | [0, 1] | 0.12 | F1 over the changed-file sets (not Jaccard — missing an oracle file is worse than touching one extra). | | `region_overlap` | [0, 1] | 0.20 | For each oracle hunk, did the predicted diff edit a line within 5 lines of that hunk in the same file? Strongest spatial-localization signal. | | `similarity` | [0, 1] | 0.10 | `difflib.SequenceMatcher` ratio over `+`/`-` lines only (no free credit for unchanged context). | -| `llm_judge` | [0, 1] or null | 0.50 | Haiku rates "does this patch logically address the issue described?" Most informative semantic signal. Null on missing API key / network error → remaining weights are re-normalized. | +| `llm_judge` | [0, 1] or null | 0.50 | An LLM rates "does this patch logically address the issue described?" — Anthropic Haiku by default, or any OpenAI-compatible server (vLLM, Ollama, a gateway) via `R2E_JUDGE_ENDPOINT` + `R2E_JUDGE_MODEL`. Most informative semantic signal. Null on missing API key / network error → remaining weights are re-normalized. | Final reward is clipped to `[0, 1]`. A **catastrophic-size hard cap** clamps the final to ≤ 0.40 when `size_sanity < 0.10` — stops a charitable judge from inflating scores on patches that are wildly the wrong size. @@ -171,8 +171,10 @@ repo2rlenv generate \ harbor run -p ./datasets/click-prdiff -a oracle --env docker -n 1 # Run it through harbor with a real agent. -# The verifier's LLM judge also needs an API key — pass via --ve so it -# reaches the verifier container (the --ae key only reaches the agent). +# The verifier's LLM judge also needs credentials — pass via --ve so they +# reach the verifier container (the --ae key only reaches the agent). +# Default judge: Anthropic Haiku via ANTHROPIC_API_KEY. Self-hosted judge: +# R2E_JUDGE_ENDPOINT + R2E_JUDGE_MODEL, no key needed (Example 3). # Example 1: claude-code + Sonnet 4.6 (what we used to verify the # reference dataset). @@ -192,6 +194,20 @@ harbor run \ --ve ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ --env docker -n 1 +# Example 3: a self-hosted judge. `vllm serve Qwen/Qwen3.5-4B --host 0.0.0.0 +# --port 8000` on the host, then route the verifier to it; ANTHROPIC_API_KEY +# is not needed and is never sent there. +harbor run \ + -p ./datasets/click-prdiff \ + -a claude-code -m anthropic/claude-sonnet-4-6 \ + --ae ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + --ve R2E_JUDGE_ENDPOINT=http://host.docker.internal:8000/v1 \ + --ve R2E_JUDGE_MODEL=Qwen/Qwen3.5-4B \ + --env docker -n 1 +# ^ host.docker.internal is a Docker Desktop name (macOS / Windows / WSL2). +# On a bare Linux daemon it does not resolve: use the host's LAN IP +# instead, e.g. --ve R2E_JUDGE_ENDPOINT=http://$(hostname -I | cut -d' ' -f1):8000/v1 + # Harbor ships 25+ agent harnesses you can swap in here: # claude-code · openhands / openhands-sdk · codex · aider · gemini-cli # copilot-cli · opencode · cursor-cli · qwen-coder · kimi-cli · goose diff --git a/docs/reference/ENV.md b/docs/reference/ENV.md index 46d97b2..af3cc25 100644 --- a/docs/reference/ENV.md +++ b/docs/reference/ENV.md @@ -81,8 +81,10 @@ The diff-similarity verifier baked into every `pr_diff` task is configurable at | `R2E_W_REGION` | Weight for the *region overlap* component. | wired in source | | `R2E_W_SIM` | Weight for the *changes-only similarity* component. | wired in source | | `R2E_W_JUDGE` | Weight for the *LLM-as-judge* semantic-correctness component. | wired in source | -| `R2E_JUDGE_MODEL` | Override the judge model (LiteLLM-qualified name). | claude-haiku | -| `ANTHROPIC_API_KEY` | Required for the LLM-judge component; the verifier degrades gracefully (records `status=no_api_key`) when unset, so the other five components still score. | +| `R2E_JUDGE_MODEL` | The judge model, as the serving API names it (a bare model id, not a LiteLLM `provider/model` string — the verifier is stdlib-only and does not go through LiteLLM). Required when `R2E_JUDGE_ENDPOINT` is set. | `claude-haiku-4-5-20251001` | +| `R2E_JUDGE_ENDPOINT` | Base URL of an OpenAI-compatible server to use as the judge instead of Anthropic — vLLM, Ollama, llama.cpp, a gateway. The verifier posts to `/chat/completions` at temperature 0 (small local models are noisy judges at their default sampling temperature). From inside the verifier container a model on the host is `http://host.docker.internal:8000/v1` on Docker Desktop (macOS / Windows / WSL2); on a bare Linux daemon that name does not resolve, so use the host's LAN IP (`hostname -I`) with the server bound to `0.0.0.0`. | unset (Anthropic) | +| `R2E_JUDGE_API_KEY` | Bearer token sent to `R2E_JUDGE_ENDPOINT`. Optional: self-hosted servers ignore it, so a placeholder is sent when unset. `ANTHROPIC_API_KEY` is never forwarded to a custom endpoint. | unset | +| `ANTHROPIC_API_KEY` | Required for the LLM-judge component on the default Anthropic route; the verifier degrades gracefully (records `judge_status=no_api_key`) when unset, so the other five components still score. Ignored when `R2E_JUDGE_ENDPOINT` is set. | ## UI / logging diff --git a/docs/reference/REWARD_SCHEMA.md b/docs/reference/REWARD_SCHEMA.md index b2ed7b5..1e058fe 100644 --- a/docs/reference/REWARD_SCHEMA.md +++ b/docs/reference/REWARD_SCHEMA.md @@ -41,7 +41,8 @@ full breakdown for analysis, filtering, and debugging. "similarity": 0.10, "llm_judge": 0.50 }, - "judge_model": "anthropic/claude-haiku-4-5-20251001", + "judge_model": "claude-haiku-4-5-20251001", + "judge_endpoint": null, "judge_status": "ok", "capped": false } @@ -59,13 +60,16 @@ full breakdown for analysis, filtering, and debugging. | `components.similarity` | [0, 1] | `difflib.SequenceMatcher` ratio over `+`/`-` lines only (no credit for context lines). | | `components.llm_judge` | [0, 1] or `null` | LLM semantic judge ("does this address the issue?"). `null` when disabled or API key absent — weight redistributed to remaining components. | | `weights` | object | Effective per-component weights (overridable via `R2E_W_*` env vars). | -| `judge_model` | string or `null` | Model used for `llm_judge`, or `null` if judge was skipped. | -| `judge_status` | string | `"ok"` \| `"no_api_key"` \| `"error"` \| `"timeout"` | +| `judge_model` | string or `null` | Model used for `llm_judge` as the serving API names it (e.g. `claude-haiku-4-5-20251001`, `Qwen/Qwen3.5-4B`), or `null` if the judge did not score. | +| `judge_endpoint` | string or `null` | The OpenAI-compatible server the judge was routed to (`R2E_JUDGE_ENDPOINT`), or `null` on the default Anthropic route. | +| `judge_status` | string | `"ok"` \| `"no_api_key"` \| `"no_judge_model"` \| `"empty_predicted"` \| `"network"` \| `"parse"` \| `"missing_score"` | | `capped` | bool | `true` if the hard size-sanity cap was applied (`reward` forced to ≤ 0.40). | **Weight override env vars** (set inside the verifier container via `--ve`): `R2E_W_FORMAT`, `R2E_W_SIZE`, `R2E_W_FILE`, `R2E_W_REGION`, `R2E_W_SIM`, `R2E_W_JUDGE` +**Judge routing env vars** (also via `--ve`): `R2E_JUDGE_MODEL`, `R2E_JUDGE_ENDPOINT`, `R2E_JUDGE_API_KEY` — see [`ENV.md`](./ENV.md#pr_diff-reward-tuning). + --- ## `pr_runtime` · `commit_runtime` · `cve_patches` diff --git a/docs/rfcs/0001-pr-diff.md b/docs/rfcs/0001-pr-diff.md index 2b66a68..b99aade 100644 --- a/docs/rfcs/0001-pr-diff.md +++ b/docs/rfcs/0001-pr-diff.md @@ -60,7 +60,7 @@ The starting point for the project. Datasets of merged PR diffs are the closest ## LLM use -- **`at verify` (per scoring)** — one Anthropic Haiku call per agent invocation, weight 0.50. Graceful degradation on missing API key: `judge_status=no_api_key`, other 5 components renormalize. +- **`at verify` (per scoring)** — one judge call per agent invocation (Anthropic Haiku by default; any OpenAI-compatible server via `R2E_JUDGE_ENDPOINT`), weight 0.50. Graceful degradation on missing API key: `judge_status=no_api_key`, other 5 components renormalize. - **No bootstrap LLM** — the thin env doesn't need it. - **Cost order-of-magnitude** — ~$0.001-$0.005 per scoring call. A 100-agent-run × 100-task eval ≈ $10-50. @@ -72,7 +72,7 @@ The starting point for the project. Datasets of merged PR diffs are the closest ## Dependencies - No reuse; `pr_diff` is the *base* pipeline. Later pipelines borrow its Dockerfile-baking + verifier pattern. -- Stdlib + `difflib`. LLM judge via LiteLLM. +- Stdlib + `difflib`. LLM judge via `urllib` (the verifier is baked into the image and stays stdlib-only — it does not go through LiteLLM). ## Alternatives considered diff --git a/src/repo2rlenv/hub.py b/src/repo2rlenv/hub.py index 248427d..b5d3230 100644 --- a/src/repo2rlenv/hub.py +++ b/src/repo2rlenv/hub.py @@ -167,9 +167,11 @@ def _reward_doc_for(pipeline: str) -> str: "The reward is a **6-component diff-similarity** score " "(format / size / file-targeting / region-overlap / changes-only " "similarity / LLM-judge). The `--ve ANTHROPIC_API_KEY=...` verifier-env " - "pass enables the LLM-judge component; without it the verifier still " - "produces a valid score with `llm_judge: null` and the deterministic " - "weights renormalized. Full breakdown in `/logs/verifier/reward-details.json`." + "pass enables the LLM-judge component (or `--ve R2E_JUDGE_ENDPOINT=...` " + "plus `--ve R2E_JUDGE_MODEL=...` to use a self-hosted judge); without " + "either, the verifier still produces a valid score with `llm_judge: null` " + "and the deterministic weights renormalized. Full breakdown in " + "`/logs/verifier/reward-details.json`." ) return ( "The reward function ships inside the task (`tests/test.sh` + verifier); " diff --git a/src/repo2rlenv/pipelines/_pr_diff_verifier.py b/src/repo2rlenv/pipelines/_pr_diff_verifier.py index 1dcce76..b71f713 100644 --- a/src/repo2rlenv/pipelines/_pr_diff_verifier.py +++ b/src/repo2rlenv/pipelines/_pr_diff_verifier.py @@ -17,7 +17,9 @@ (strongest spatial-localization signal) similarity ([0, 1]) — SequenceMatcher ratio over +/- lines only (no free credit for context lines) - llm_judge ([0, 1] or null) — Haiku rates "does this address the issue?" + llm_judge ([0, 1] or null) — an LLM rates "does this address the issue?" + Anthropic Haiku by default; any OpenAI- + compatible server via ``R2E_JUDGE_ENDPOINT``. null on API failure / missing key → remaining weights are re-normalized @@ -29,6 +31,14 @@ ``R2E_W_SIM`` / ``R2E_W_JUDGE`` (pass via harbor ``--ve`` so they reach the verifier container). +Judge routing, also via ``--ve``: ``R2E_JUDGE_MODEL`` picks the model +(default Haiku); ``R2E_JUDGE_ENDPOINT`` points at an OpenAI-compatible +server (vLLM, Ollama, a gateway) and switches the request to +``/chat/completions`` with a bearer token from +``R2E_JUDGE_API_KEY`` — or a placeholder when unset, since self-hosted +servers ignore it. ``ANTHROPIC_API_KEY`` is only read on the default +route and is never sent to a custom endpoint. + Final reward is clipped to [0, 1] and additionally clamped to ≤ 0.40 when ``size_sanity < 0.10`` (catastrophic-size hard cap — stops a charitable judge from inflating scores on wildly wrong-sized patches). @@ -268,6 +278,10 @@ def similarity(oracle: str, predicted: str) -> float: _DEFAULT_JUDGE_MODEL = "claude-haiku-4-5-20251001" +_ANTHROPIC_MESSAGES_URL = "https://api.anthropic.com/v1/messages" +# Bearer token for a self-hosted judge when R2E_JUDGE_API_KEY is unset. vLLM / +# Ollama ignore it; the OpenAI wire format still wants one. Same value llm.py uses. +_PLACEHOLDER_API_KEY = "EMPTY" def llm_judge( @@ -278,17 +292,25 @@ def llm_judge( api_key: str, model: str = _DEFAULT_JUDGE_MODEL, timeout: int = 60, + endpoint: str | None = None, ) -> tuple[float | None, str]: """Return ``(score, status)``. ``score`` is a float in [0, 1] on success, ``None`` on failure. ``status`` is a short string: ``"ok"`` / ``"no_api_key"`` / - ``"empty_predicted"`` / ``"network"`` / ``"parse"`` / - ``"missing_score"``. Caller redistributes the judge weight if score - is None. + ``"no_judge_model"`` / ``"empty_predicted"`` / ``"network"`` / + ``"parse"`` / ``"missing_score"``. Caller redistributes the judge + weight if score is None. + + ``endpoint`` — base URL of an OpenAI-compatible server. When set, the + request goes to ``/chat/completions`` with ``api_key`` as a + bearer token, at temperature 0, and ``model`` must be given explicitly; + otherwise it goes to Anthropic's Messages API unchanged. """ if not api_key: return None, "no_api_key" + if endpoint and not model: + return None, "no_judge_model" if not predicted.strip(): return 0.0, "empty_predicted" @@ -298,23 +320,31 @@ def llm_judge( oracle=oracle[:4000], predicted=predicted[:4000], ) - body = json.dumps( - { - "model": model, - "max_tokens": 200, - "messages": [{"role": "user", "content": prompt}], + payload: dict = { + "model": model, + "max_tokens": 200, + "messages": [{"role": "user", "content": prompt}], + } + if endpoint: + # Small self-hosted models are noisy judges at their default sampling + # temperature (same input scoring 0.0 one call and 1.0 the next); pin + # them to greedy. The Anthropic route keeps its defaults — the weights + # were calibrated against them. + payload["temperature"] = 0 + url = endpoint.rstrip("/") + "/chat/completions" + headers = { + "authorization": f"Bearer {api_key}", + "content-type": "application/json", } - ).encode("utf-8") - req = urllib.request.Request( - "https://api.anthropic.com/v1/messages", - data=body, - headers={ + else: + url = _ANTHROPIC_MESSAGES_URL + headers = { "x-api-key": api_key, "anthropic-version": "2023-06-01", "content-type": "application/json", - }, - method="POST", - ) + } + body = json.dumps(payload).encode("utf-8") + req = urllib.request.Request(url, data=body, headers=headers, method="POST") try: with urllib.request.urlopen(req, timeout=timeout) as resp: raw = resp.read().decode("utf-8") @@ -323,7 +353,12 @@ def llm_judge( try: payload = json.loads(raw) - text = payload["content"][0]["text"] + if endpoint: + text = payload["choices"][0]["message"]["content"] + else: + text = payload["content"][0]["text"] + if not isinstance(text, str): + raise TypeError("judge response text is not a string") except (json.JSONDecodeError, KeyError, IndexError, TypeError): return None, "parse" @@ -423,6 +458,25 @@ def _read_weights_from_env() -> dict[str, float]: return out +def _judge_config_from_env() -> tuple[str, str, str | None]: + """``(api_key, model, endpoint)`` for the judge, from the verifier env. + + With ``R2E_JUDGE_ENDPOINT`` set, the key comes from ``R2E_JUDGE_API_KEY`` + (placeholder when unset) and ``ANTHROPIC_API_KEY`` is never forwarded to + the custom server. ``R2E_JUDGE_MODEL`` has no default on that route (a + self-hosted server won't have Haiku): when unset, ``model`` comes back + as ``""`` and ``llm_judge`` reports ``no_judge_model`` instead of + calling out. Without an endpoint the default Anthropic route is + unchanged. + """ + endpoint = os.environ.get("R2E_JUDGE_ENDPOINT", "").strip() or None + model = os.environ.get("R2E_JUDGE_MODEL", "").strip() + if endpoint: + api_key = os.environ.get("R2E_JUDGE_API_KEY", "").strip() or _PLACEHOLDER_API_KEY + return api_key, model, endpoint + return os.environ.get("ANTHROPIC_API_KEY", "").strip(), model or _DEFAULT_JUDGE_MODEL, None + + def main(argv: list[str] | None = None) -> int: args = argv if argv is not None else sys.argv[1:] if len(args) < 3: @@ -435,8 +489,7 @@ def main(argv: list[str] | None = None) -> int: oracle = _read_or_empty(args[0]) predicted = _read_or_empty(args[1]) instruction = _read_or_empty(args[2]) - api_key = os.environ.get("ANTHROPIC_API_KEY", "").strip() - judge_model = os.environ.get("R2E_JUDGE_MODEL", _DEFAULT_JUDGE_MODEL) + api_key, judge_model, judge_endpoint = _judge_config_from_env() fv = format_valid(predicted) ss = size_sanity(oracle, predicted) @@ -449,6 +502,7 @@ def main(argv: list[str] | None = None) -> int: predicted=predicted, api_key=api_key, model=judge_model, + endpoint=judge_endpoint, ) components: dict[str, float | None] = { @@ -476,6 +530,7 @@ def main(argv: list[str] | None = None) -> int: "components": {k: (None if v is None else round(v, 6)) for k, v in components.items()}, "weights": {k: round(v, 6) for k, v in weights.items()}, "judge_model": judge_model if judge_status == "ok" else None, + "judge_endpoint": judge_endpoint, "judge_status": judge_status, } diff --git a/src/repo2rlenv/pipelines/pr_diff.py b/src/repo2rlenv/pipelines/pr_diff.py index f87e3aa..a7ed182 100644 --- a/src/repo2rlenv/pipelines/pr_diff.py +++ b/src/repo2rlenv/pipelines/pr_diff.py @@ -186,8 +186,8 @@ def build_pr_diff_environment_dockerfile( No bootstrap LLM agent — just python:3.12-slim + git + the repo checked out at ``base_commit``. The oracle diff, the instruction, AND the verifier source are all base64-baked into the image so the verifier - runs offline with only the Anthropic API call (for the LLM judge) as - its outbound dep. + runs offline with only the LLM-judge call (Anthropic by default, or + the server named by ``R2E_JUDGE_ENDPOINT``) as its outbound dep. The clone uses an optional ``GITHUB_TOKEN`` or ``GITLAB_TOKEN`` build arg, selected by host. Public repos need no arg. The authenticated diff --git a/tests/test_pipeline_pr_diff.py b/tests/test_pipeline_pr_diff.py index 9de0ec8..d74d390 100644 --- a/tests/test_pipeline_pr_diff.py +++ b/tests/test_pipeline_pr_diff.py @@ -351,3 +351,21 @@ def test_dockerfile_supports_private_repo_build_arg() -> None: assert "remote set-url origin https://github.com/myorg/private-repo.git" in df # The token itself must never appear literally baked anywhere. assert "ghp_" not in df + + +def test_verifier_source_is_stdlib_only() -> None: + """The verifier is baked into a bare python:3.12-slim image — nothing but stdlib may be imported.""" + import ast + import sys + + from repo2rlenv.pipelines.pr_diff import _verifier_source + + tree = ast.parse(_verifier_source()) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + imported.add(node.module.split(".")[0]) + non_stdlib = sorted(imported - sys.stdlib_module_names) + assert non_stdlib == [], f"verifier imports non-stdlib modules: {non_stdlib}" diff --git a/tests/test_pr_diff_verifier.py b/tests/test_pr_diff_verifier.py index 9ec1db9..4800673 100644 --- a/tests/test_pr_diff_verifier.py +++ b/tests/test_pr_diff_verifier.py @@ -7,11 +7,13 @@ from __future__ import annotations +import json from unittest import mock import pytest from repo2rlenv.pipelines._pr_diff_verifier import ( + _judge_config_from_env, _normalize_changes_only, combine, file_paths, @@ -296,6 +298,160 @@ def test_llm_judge_clamps_out_of_range_score() -> None: assert status == "ok" +# --------------------------------------------------------------------------- +# llm_judge — routing: Anthropic by default, OpenAI-compatible via endpoint +# --------------------------------------------------------------------------- + + +_ENDPOINT = "http://127.0.0.1:8000/v1" +_OPENAI_STYLE_OK = ( + b'{"choices": [{"message": {"role": "assistant", ' + b'"content": "{\\"score\\": 0.6, \\"reasoning\\": \\"ok\\"}"}}]}' +) + + +def _judge_via(endpoint, response: bytes, **kw): + """Run llm_judge with urlopen mocked; return (score, status, Request).""" + with mock.patch("urllib.request.urlopen") as m: + m.return_value.__enter__.return_value.read.return_value = response + score, status = llm_judge( + instruction="i", + oracle="diff --git a/x b/x\n", + predicted="diff --git a/x b/x\n+fix\n", + api_key=kw.pop("api_key", "EMPTY"), + endpoint=endpoint, + **kw, + ) + req = m.call_args.args[0] + return score, status, req + + +def test_llm_judge_default_route_is_anthropic() -> None: + fake = b'{"content": [{"text": "{\\"score\\": 0.5}"}]}' + score, status, req = _judge_via(None, fake, api_key="sk-ant", model="claude-x") + assert status == "ok" and score == pytest.approx(0.5) + assert req.full_url == "https://api.anthropic.com/v1/messages" + assert req.get_header("X-api-key") == "sk-ant" + assert req.get_header("Authorization") is None + assert "temperature" not in json.loads(req.data) # default route untouched + + +def test_llm_judge_endpoint_uses_openai_compatible_route() -> None: + score, status, req = _judge_via(_ENDPOINT, _OPENAI_STYLE_OK, model="Qwen/Qwen3.5-4B") + assert status == "ok" and score == pytest.approx(0.6) + assert req.full_url == _ENDPOINT + "/chat/completions" + assert req.get_header("Authorization") == "Bearer EMPTY" + assert req.get_header("X-api-key") is None + body = json.loads(req.data) + assert body["model"] == "Qwen/Qwen3.5-4B" + assert body["messages"][0]["role"] == "user" + assert body["temperature"] == 0 # greedy: small local judges are noisy otherwise + + +def test_llm_judge_endpoint_trailing_slash_and_explicit_key() -> None: + _, status, req = _judge_via(_ENDPOINT + "/", _OPENAI_STYLE_OK, api_key="srv", model="m") + assert status == "ok" + assert req.full_url == _ENDPOINT + "/chat/completions" + assert req.get_header("Authorization") == "Bearer srv" + + +def test_llm_judge_endpoint_requires_model() -> None: + score, status = llm_judge( + instruction="i", oracle="o", predicted="p", api_key="EMPTY", model="", endpoint=_ENDPOINT + ) + assert score is None + assert status == "no_judge_model" + + +def test_llm_judge_endpoint_anthropic_shaped_reply_is_parse_error() -> None: + """A server answering in Anthropic's shape on the OpenAI route is a parse failure, not a crash.""" + anthropic_shaped = b'{"content": [{"text": "{\\"score\\": 0.9}"}]}' + score, status, _ = _judge_via(_ENDPOINT, anthropic_shaped, model="m") + assert score is None + assert status == "parse" + + +def test_llm_judge_endpoint_null_content_is_parse_error() -> None: + """Thinking-mode servers can return content=null with the text elsewhere.""" + null_content = b'{"choices": [{"message": {"content": null, "reasoning_content": "..."}}]}' + score, status, _ = _judge_via(_ENDPOINT, null_content, model="m") + assert score is None + assert status == "parse" + + +def test_llm_judge_endpoint_network_error() -> None: + import urllib.error + + with mock.patch("urllib.request.urlopen", side_effect=urllib.error.URLError("refused")): + score, status = llm_judge( + instruction="i", + oracle="o", + predicted="p", + api_key="EMPTY", + model="m", + endpoint=_ENDPOINT, + ) + assert score is None + assert status == "network" + + +# --------------------------------------------------------------------------- +# _judge_config_from_env — what main() hands to llm_judge +# --------------------------------------------------------------------------- + + +_JUDGE_VARS = ("ANTHROPIC_API_KEY", "R2E_JUDGE_MODEL", "R2E_JUDGE_ENDPOINT", "R2E_JUDGE_API_KEY") + + +def _env(monkeypatch: pytest.MonkeyPatch, **values: str) -> None: + for name in _JUDGE_VARS: + monkeypatch.delenv(name, raising=False) + for name, value in values.items(): + monkeypatch.setenv(name, value) + + +def test_judge_config_default_route_no_key(monkeypatch: pytest.MonkeyPatch) -> None: + _env(monkeypatch) + assert _judge_config_from_env() == ("", "claude-haiku-4-5-20251001", None) + + +def test_judge_config_default_route_with_key_and_model(monkeypatch: pytest.MonkeyPatch) -> None: + _env(monkeypatch, ANTHROPIC_API_KEY=" sk-ant ", R2E_JUDGE_MODEL="claude-x") + assert _judge_config_from_env() == ("sk-ant", "claude-x", None) + + +def test_judge_config_endpoint_withholds_anthropic_key(monkeypatch: pytest.MonkeyPatch) -> None: + _env( + monkeypatch, + ANTHROPIC_API_KEY="sk-ant", + R2E_JUDGE_ENDPOINT=_ENDPOINT, + R2E_JUDGE_MODEL="Qwen/Qwen3.5-4B", + ) + api_key, model, endpoint = _judge_config_from_env() + assert api_key == "EMPTY" + assert (model, endpoint) == ("Qwen/Qwen3.5-4B", _ENDPOINT) + + +def test_judge_config_endpoint_explicit_key(monkeypatch: pytest.MonkeyPatch) -> None: + _env(monkeypatch, R2E_JUDGE_ENDPOINT=_ENDPOINT, R2E_JUDGE_MODEL="m", R2E_JUDGE_API_KEY="srv") + assert _judge_config_from_env() == ("srv", "m", _ENDPOINT) + + +def test_judge_config_endpoint_without_model_has_no_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _env(monkeypatch, R2E_JUDGE_ENDPOINT=_ENDPOINT) + api_key, model, endpoint = _judge_config_from_env() + assert api_key == "EMPTY" + assert model == "" # llm_judge turns this into status "no_judge_model" + assert endpoint == _ENDPOINT + + +def test_judge_config_blank_endpoint_means_default_route(monkeypatch: pytest.MonkeyPatch) -> None: + _env(monkeypatch, R2E_JUDGE_ENDPOINT=" ", ANTHROPIC_API_KEY="sk-ant") + assert _judge_config_from_env() == ("sk-ant", "claude-haiku-4-5-20251001", None) + + # --------------------------------------------------------------------------- # combine — weight redistribution when judge is None # ---------------------------------------------------------------------------