From cc238d61bb255156bc5679dff8b15274a4fc5fe9 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sun, 23 Aug 2026 00:38:26 +0800 Subject: [PATCH 01/10] =?UTF-8?q?feat(tasks):=20add=20Ag-LiveCodeBench-X?= =?UTF-8?q?=20=E2=80=94=20competitive=20programming=20in=20any=20language?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LiveCodeBench v5's 499 stdin/stdout problems, prepared by Agnostics (ICLR 2026, arXiv:2508.04865) so the same problem and the same test cases score a solution in any language. Ships `status="experimental"`: faithful on the axes below, but no run has been aligned against a published number. The language is a task argument, not a property of the data, so `language` is required with no default — the benchmark measures a different thing per language, and a silent default would let two runs of one task name mean different things. `container_lang` (the verifier tag) defaults to `language`'s first word lowercased and stays overridable, because upstream keeps the two independent and warns nothing checks them against each other. Both are recorded in `report.json`, which the task name cannot do. The prompt is a frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem), captured by running it: two of its properties are invisible in the signature — the instructions are re-inserted into the system message still carrying the source docstring's 8-space indent, and the field list is stripped as a block so the first line keeps a trailing space and the last loses it. Frozen rather than depending on DSPy at runtime, so a patch release cannot re-word the fleet's prompts; pinned by sha256 in the tests. Response parsing and code extraction are reimplemented, not vendored (upstream ships no license), including an upstream offset bug that leaks `]]` out of an indented field header. Enumerated divergence with a consequence: upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt and this port makes one call, so a reply missing either marker scores 0 here. The number is a lower bound whose gap is exactly `n_unextracted`. Execution goes through the code-evaluator's new `agnostics` source rather than from the task process, so sieval keeps one code-execution boundary and a client cannot name an arbitrary container to run. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/datasets/__init__.pyi | 6 + sieval/datasets/ag_livecodebench_x.py | 87 +++ sieval/meta/index.json | 47 ++ sieval/tasks/__init__.pyi | 4 + sieval/tasks/ag_livecodebench_x_0shot_gen.py | 548 ++++++++++++++++++ .../test_ag_livecodebench_x_0shot_gen.py | 450 ++++++++++++++ vendor/code-evaluator/README.md | 66 ++- vendor/code-evaluator/VENDORED.md | 45 ++ vendor/code-evaluator/app/exec_agnostics.py | 152 +++++ vendor/code-evaluator/app/server.py | 54 ++ 10 files changed, 1456 insertions(+), 3 deletions(-) create mode 100644 sieval/datasets/ag_livecodebench_x.py create mode 100644 sieval/tasks/ag_livecodebench_x_0shot_gen.py create mode 100644 tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py create mode 100644 vendor/code-evaluator/app/exec_agnostics.py diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index 79ef2bd0..fa8d0d92 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -9,6 +9,10 @@ from .advanced_if import ( AdvancedIFDataset, AdvancedIFDatasetSample, ) +from .ag_livecodebench_x import ( + AgLiveCodeBenchXDataset, + AgLiveCodeBenchXDatasetSample, +) from .agieval import ( AGIEvalDataset, AGIEvalDatasetSample, @@ -259,6 +263,8 @@ __all__ = [ "ARCEasyDatasetSample", "AdvancedIFDataset", "AdvancedIFDatasetSample", + "AgLiveCodeBenchXDataset", + "AgLiveCodeBenchXDatasetSample", "Apex2025Dataset", "Apex2025DatasetSample", "ApexShortlist2025Dataset", diff --git a/sieval/datasets/ag_livecodebench_x.py b/sieval/datasets/ag_livecodebench_x.py new file mode 100644 index 00000000..4185894b --- /dev/null +++ b/sieval/datasets/ag_livecodebench_x.py @@ -0,0 +1,87 @@ +"""Ag-LiveCodeBench-X — the multi-language LiveCodeBench subset from Agnostics. + +Upstream (NUPRL, `Agnostics `_) took +LiveCodeBench v5's 880 problems, kept the **499 that specify and check their +answer purely through stdin/stdout**, and dropped the 381 that ship Python +starter code. What is left is language-agnostic: the same problem statement and +the same test cases score a solution written in any language, which is why the +set can be pointed at Lua, R, Julia, OCaml or Fortran without re-authoring +anything. + +Three columns and no more — there is no ``starter_code``, no ``metadata`` and no +``public_test_cases`` split here, so every case is a private one and the +call-based (``fn_name``) grading path LiveCodeBench also carries cannot arise. +``private_test_cases`` keeps LiveCodeBench's own encoding (base64 -> zlib -> +pickle -> JSON); decoded, the 499 problems are ~8 GB of test data, which is why +it travels compressed. + +Upstream publishes **no license**: the Hub card carries no license field and the +scripts repo has no ``LICENSE`` file, so ``license`` is ``None`` rather than a +guess. See the task's ``reference_impl.notes`` for what that costs in port +fidelity. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +AG_LIVECODEBENCH_X_REVISION = "15bef6bf966893163ea9d22f25b7c58fec06a949" + + +class AgLiveCodeBenchXDatasetSample(TypedDict): + question_id: str + question_content: str + private_test_cases: str + + +@sieval_dataset( + name="ag_livecodebench_x", + display_name="Ag-LiveCodeBench-X", + description=( + "LiveCodeBench v5's 499 stdin/stdout problems, prepared by Agnostics " + "for any language." + ), + source=f"hf:nuprl/Ag-LiveCodeBench-X@{AG_LIVECODEBENCH_X_REVISION}", + categories=( + Category(Level1Category.CODE, "CodeGeneration"), + Category(Level1Category.CODE, "MultiLanguageSupport"), + ), + # No `python` tag, unlike `livecodebench_code_generation`: the target + # language is the task's `language` argument, not a property of the data. + tags=("english", "code-exec"), + # Upstream declares none (no Hub license field, no LICENSE in the scripts + # repo). `None` is the accurate record; do not infer one from LiveCodeBench, + # whose own label ('cc') covers a different artifact. + license=None, +) +class AgLiveCodeBenchXDataset(Dataset[AgLiveCodeBenchXDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + # One call for both a Hub id and a staged directory: the release is plain + # parquet under `data/`, with no loader script and no config to select, + # so `load_dataset` resolves either form on its own. No `trust_remote_code` + # for the same reason -- unlike `livecodebench_code_generation`, whose Hub + # copy ships a builder script. + dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs)) + + # A window that filters everything away would otherwise be reported as a + # score of 0.0 rather than as a mistake. + if any(len(split) == 0 for split in dataset.values()): + raise ValueError( + "Ag-LiveCodeBench-X dataset is empty after loading " + f"(name_or_path={name_or_path!r}); the release has a single " + "'test' split of 499 problems." + ) + + return dataset diff --git a/sieval/meta/index.json b/sieval/meta/index.json index fbecff08..ff6ef81d 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -45,6 +45,31 @@ "license": "CC-BY-NC-4.0", "checksums": {} }, + { + "name": "ag_livecodebench_x", + "display_name": "Ag-LiveCodeBench-X", + "description": "LiveCodeBench v5's 499 stdin/stdout problems, prepared by Agnostics for any language.", + "source": [ + "hf:nuprl/Ag-LiveCodeBench-X@15bef6bf966893163ea9d22f25b7c58fec06a949" + ], + "categories": [ + { + "level1": "Code", + "level2": "CodeGeneration" + }, + { + "level1": "Code", + "level2": "MultiLanguageSupport" + } + ], + "tags": [ + "english", + "code-exec" + ], + "deps_group": null, + "license": null, + "checksums": {} + }, { "name": "agieval", "display_name": "AGIEval", @@ -1445,6 +1470,28 @@ "status": "experimental", "reference_kind": "procedure" }, + { + "name": "ag_livecodebench_x_0shot_gen", + "display_name": "Ag-LiveCodeBench-X (0-shot)", + "description": "Competitive programming in an arbitrary language, graded by the Agnostics verifier.", + "dataset": "ag_livecodebench_x", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "code-exec", + "multi-language" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "nuprl/Ag-LiveCodeBench-X", + "url": "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py", + "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, same protocol and same image, different process boundary; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). No sieval run has been aligned against a published number yet, which is what status='experimental' records." + }, + "status": "experimental", + "reference_kind": "procedure" + }, { "name": "agieval_0shot_gen", "display_name": "AGIEval (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 12a6a673..79df62e8 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -7,6 +7,9 @@ from .aa_lcr_0shot_gen import ( from .advanced_if_0shot_gen import ( AdvancedIFZeroShotGenTask, ) +from .ag_livecodebench_x_0shot_gen import ( + AgLiveCodeBenchXZeroShotGenTask, +) from .agieval_0shot_gen import ( AGIEvalZeroShotGenTask, ) @@ -247,6 +250,7 @@ __all__ = [ "ARCEasyFewShotClpTask", "ARCEasyFewShotPplTask", "AdvancedIFZeroShotGenTask", + "AgLiveCodeBenchXZeroShotGenTask", "Apex2025ZeroShotGenTask", "ApexShortlist2025ZeroShotGenTask", "BRUMO2025ZeroShotGenTask", diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py new file mode 100644 index 00000000..a1afc474 --- /dev/null +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -0,0 +1,548 @@ +"""Ag-LiveCodeBench-X — 0-shot competitive programming in an arbitrary language. + +Port of `nuprl/Ag-LiveCodeBench-X `_, +the multi-language LiveCodeBench prepared by Agnostics (Boruch-Gruszecki et al., +ICLR 2026; ``arXiv:2508.04865``). The 499 problems are LiveCodeBench v5's +stdin/stdout subset, so the *same* problem and the *same* test cases score a +solution in any language: which language is asked for is an argument, not a +property of the data. + +Two knobs, deliberately independent, because upstream keeps them independent: + +* ``language`` is a free-form string spliced into the prompt (``"Lua"``, + ``"Julia 1.10"``, ``"Lua /nothink"`` — upstream's own README suggests appending + a version or a thinking-disable directive there). +* ``container_lang`` selects the verifier the evaluator runs. Upstream warns that + nothing checks the two against each other; the default here derives the tag + from ``language``'s first word, lowercased, which is a convenience and not a + guarantee. Override it whenever the derivation is wrong (``"C++"`` is not a + container tag). + +Both are recorded in ``report()``, because the task name cannot hold them: one +registered task measures a different quantity per language, and a leaderboard +column that does not say which language it ran is not comparable with anything. + +**The prompt is a frozen DSPy rendering, not a string upstream wrote.** Upstream +declares a ``dspy.Signature`` and wraps it in ``dspy.ChainOfThought``; the bytes +that reach the model are whatever ``dspy==3.0.0b2``'s default ``ChatAdapter`` +renders from that declaration, pinned by upstream's ``exclude-newer`` of +2025-08-05. Reproducing it by reading the signature is not possible — two of its +properties are only observable by running it: + +* the instructions are re-inserted into the system message **still carrying the + source docstring's 8-space indentation**, while ``Signature.instructions`` + itself is dedented; +* the field list is assembled with a trailing ``": "`` per line and then + stripped as a block, so the *first* field line keeps its trailing space and the + *last* one loses it. + +So the rendering was captured once from ``dspy==3.0.0b2`` and frozen below. +Freezing is the point rather than a shortcut: a runtime DSPy dependency would +let a patch release silently re-word every prompt in the fleet, which is the +opposite of what a pinned benchmark is for. +``tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py`` pins the rendered +bytes by hash. + +**Reimplemented rather than vendored, because upstream ships no license.** The +Hub card declares none and the scripts repo has no ``LICENSE``, so nothing lands +in ``sieval/community/``: the response parser and the code extractor below are +written from observed behaviour and pinned by tests, in the shape +``sieval/community/`` is reserved for when a license permits a mirror. + +Divergences from upstream, each with what it costs: + +* **No JSONAdapter retry.** ``ChatAdapter.__call__`` catches *any* exception from + its own parse and re-issues the whole call through ``JSONAdapter`` — a second, + differently shaped prompt with a JSON response format. This port makes one + call. A reply that omits either marker therefore scores 0 here where upstream + gets one more chance, so this number is a **lower bound**, and the size of the + gap is exactly ``n_unextracted``. Read it before comparing against a published + row; upstream's own README warns that truncation at its default + ``max_tokens=5000`` is common enough to mention, and truncation is precisely + what strips the closing marker. +* **The container is run by the evaluator, not by this task.** Upstream shells + out to ``podman run ... ghcr.io/nuprl/agnostics:`` (or ``apptainer`` for + a ``.sif``) from the harness process. Here the task POSTs to sieval's + code-evaluator, which runs that same container server-side; the invocation is + the evaluator's deployment config, not a per-request field, so a client cannot + name an arbitrary image to execute. Same verifier, same protocol, different + process boundary. +* **No smoke call.** Upstream sends ``lm("Say this is a test!")`` before the run; + it measures nothing and is not reproduced. +* **No DSPy cache.** Upstream defaults it off (``--enable-dspy-cache`` is opt-in + and forces concurrency 1), so this matches the default rather than diverging. + +Sampling protocol: upstream's ``--num-completions`` defaults to **1**, and +``pass1`` divides successes by rows written — a generation that produced no +solution is still executed and still counted, so the denominator is +``DENOMINATOR_REQUESTED``. Upstream's decoding defaults are ``temperature=0.6``, +``top_p=0.95``, ``max_tokens=5000``, with the README recommending ``0.2`` / +``2048`` plus ``/nothink`` for Qwen 3. Those are model-layer settings in sieval +(``models:`` / ``infer_args``), never this task's. + +``status="experimental"``: the port is faithful on the axes above, but no run has +been aligned against a published number yet. The anchor when one is attempted is +the paper's Ag-LiveCodeBench-X table (Qwen-3 4B/8B, DeepSeek-Coder-6.7B-Instruct, +Phi-4-Mini over Lua, Julia, R, OCaml and Fortran); promoting this task to +``stable`` wants one of those cells reproduced within its noise, at the paper's +own decoding settings and language. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import base64 +import json +import os +import pickle +import re +import time +import zlib +from typing import override + +import httpx +from loguru import logger + +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + RolloutJudgement, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, + health_metrics, + sampling_report, +) +from sieval.datasets import AgLiveCodeBenchXDatasetSample + +# --- The frozen DSPy 3.0.0b2 ChatAdapter rendering ------------------------- +# +# Captured from `ChatAdapter().format(ChainOfThought(SolveProblem).predict.signature, +# [], inputs)` under upstream's pinned resolution. Written as adjacent literals +# with explicit `\n` so that the two lines which legitimately END IN A SPACE +# survive an editor and a linter -- as a triple-quoted block they would not. +# Do not reflow: every byte here is on the wire. +_SYSTEM_MESSAGE = ( + "Your input fields are:\n" + "1. `programming_language` (str): \n" + "2. `problem_statement` (str):\n" + "Your output fields are:\n" + "1. `reasoning` (str): \n" + "2. `solution` (str):\n" + "All interactions will be structured in the following way, " + "with the appropriate values filled in.\n" + "\n" + "[[ ## programming_language ## ]]\n" + "{programming_language}\n" + "\n" + "[[ ## problem_statement ## ]]\n" + "{problem_statement}\n" + "\n" + "[[ ## reasoning ## ]]\n" + "{reasoning}\n" + "\n" + "[[ ## solution ## ]]\n" + "{solution}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Solve the following programming problem using the programming " + "language that I have specified. Use ONLY the\n" + " programming language given below." +) + +# The user turn is assembled by concatenation rather than `format`: the system +# message above contains literal `{...}` placeholders that are part of the wire +# text, and a problem statement is free to contain braces of its own. +_USER_PREFIX = "[[ ## programming_language ## ]]\n" +_USER_INFIX = "\n\n[[ ## problem_statement ## ]]\n" +_USER_SUFFIX = ( + "\n\nRespond with the corresponding output fields, starting with the field " + "`[[ ## reasoning ## ]]`, then `[[ ## solution ## ]]`, and then ending with " + "the marker for `[[ ## completed ## ]]`." +) + +# `dspy.adapters.chat_adapter.field_header_pattern`, verbatim. +_FIELD_HEADER_PATTERN = re.compile(r"\[\[ ## (\w+) ## \]\]") + +# `ChainOfThought` prepends `reasoning` to the signature's own outputs, and +# ChatAdapter requires *every* output field to be present or it raises. +_OUTPUT_FIELDS = frozenset({"reasoning", "solution"}) + +# The evaluator answers with the Agnostics `result` value verbatim when the +# container ran and spoke the protocol, and with an `infra:`-prefixed code when +# it did not. Upstream's own `pass1` reports the same split as `run_error_rate`, +# reading it off a stderr suffix; the prefix is a server-side category instead, +# because a client-side classifier over free text decays silently. +# Whether a `result` counts as a pass is the evaluator's call, not read back from +# this string here -- the task reads the response's `status`. +_INFRA_PREFIX = "infra:" + + +def _render_prompt( + language: str, question_content: str +) -> tuple[dict[str, str], dict[str, str]]: + """The two turns DSPy would have sent, system first. + + A tuple rather than a list so the type stays precise: a ``list[dict[str, str]]`` + is not a ``list[JSONValue]`` (``list`` is invariant), and widening the element + type here would cost every caller the ability to index a message. + """ + return ( + {"role": "system", "content": _SYSTEM_MESSAGE}, + { + "role": "user", + "content": _USER_PREFIX + + language + + _USER_INFIX + + question_content + + _USER_SUFFIX, + }, + ) + + +def _parse_chat_adapter_fields(completion: str) -> dict[str, str] | None: + """Reproduce ``ChatAdapter.parse`` for this signature; ``None`` on its raise. + + Upstream wraps the DSPy call in a bare ``except`` that turns any parse + failure into ``solution = None``, so a raise and a miss are the same outcome + there. Faithfully reproduced quirks: + + * the **first** occurrence of a field wins; later ones are ignored; + * content on the same line as a header is kept as that field's first line; + * unknown headers (``completed`` included) and any prose before the first + header are dropped; + * an **indented** header still matches, but the inline-content slice is taken + with an offset measured on the *stripped* line and applied to the + *unstripped* one -- so ``" [[ ## solution ## ]]"`` yields a field whose + first line is the leftover ``"]]"``. That is upstream's behaviour, bug and + all, and changing it here would silently move scores. + + ``parse_value`` is not reproduced because it cannot matter: both fields are + annotated ``str``, for which it is ``str(value)``. + """ + sections: list[tuple[str | None, list[str]]] = [(None, [])] + for line in completion.splitlines(): + match = _FIELD_HEADER_PATTERN.match(line.strip()) + if match: + header = match.group(1) + remaining_content = line[match.end() :].strip() + sections.append((header, [remaining_content] if remaining_content else [])) + else: + sections[-1][1].append(line) + + fields: dict[str, str] = {} + for name, lines in sections: + if name is not None and name not in fields and name in _OUTPUT_FIELDS: + fields[name] = "\n".join(lines).strip() + + # `ChatAdapter` compares the parsed key set against the signature's and + # raises when they differ -- a reply carrying `solution` but no `reasoning` + # is a parse failure, not a partial success. + if fields.keys() != _OUTPUT_FIELDS: + return None + return fields + + +def _extract_code_from_markdown(markdown: str | None) -> str | None: + """Reproduce upstream's ``extract_code_from_markdown``. + + First fenced block wins. Note the two behaviours that are easy to + "improve" by accident: **no** fence at all means the whole reply is treated + as code, and an **unterminated** fence returns ``None``. The + ``# Example usage:`` truncation is a Python-shaped heuristic that upstream + applies to every language, so it applies here too. + """ + if markdown is None: + return None + code_block_start = markdown.find("```") + if code_block_start == -1: + # Assume that the whole string is code. + return markdown + + code_start = code_block_start + 3 + code_block_end = markdown.find("```", code_start) + if code_block_end == -1: + return None + + code = markdown[code_start:code_block_end].strip() + + if "# Example usage:" in code: + code = code.split("# Example usage:")[0] + + # Drop the fence's language tag, which `strip()` above left on line one. + first_newline = code.find("\n") + if first_newline > 0: + code = code[first_newline + 1 :] + + return code.strip() + + +def _decode_private_test_cases(text: str) -> list[dict[str, str]]: + """base64 -> zlib -> pickle -> JSON, as LiveCodeBench encodes and upstream + decodes. No plain-JSON fallback: unlike ``livecodebench/code_generation_lite``, + every row of this release is encoded, and a fallback would turn a corrupt + column into a silently empty suite.""" + cases: list[dict[str, str]] = json.loads( + pickle.loads(zlib.decompress(base64.b64decode(text.encode("utf-8")))) + ) + return cases + + +@sieval_task( + name="ag_livecodebench_x_0shot_gen", + display_name="Ag-LiveCodeBench-X (0-shot)", + description=( + "Competitive programming in an arbitrary language, graded by the " + "Agnostics verifier." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "code-exec", "multi-language"), + model_type="chat", + status="experimental", + reference_kind="procedure", + reference_impl=ReferenceImpl( + source="nuprl/Ag-LiveCodeBench-X", + url=( + "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/" + "b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py" + ), + notes=( + "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of " + "upstream's ChainOfThought(SolveProblem) -- captured, not read off " + "the signature, because the 8-space instruction indent and the " + "trailing-space asymmetry in the field list are only observable at " + "runtime; pinned by hash in the task's tests. Response parsing and " + "code extraction are REIMPLEMENTED, not vendored: upstream ships no " + "license (no Hub license field, no LICENSE file). Divergences: (1) " + "upstream's ChatAdapter retries a failed parse through JSONAdapter " + "with a differently shaped prompt, which this port does not -- so a " + "reply missing either marker scores 0 here and the gap is exactly " + "n_unextracted, making this a lower bound; (2) the Agnostics " + "verifier container is run by sieval's code-evaluator rather than " + "from the harness process, same protocol and same image, different " + "process boundary; (3) upstream's pre-run 'Say this is a test!' " + "smoke call is not reproduced. Upstream defaults: n=1 completion, " + "temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout " + "supplied per run (README uses 15s, and sends that one number as " + "BOTH the container's timeout_s and the outer process wall). No " + "sieval run has been aligned against a published number yet, which " + "is what status='experimental' records." + ), + ), +) +class AgLiveCodeBenchXZeroShotGenTask( + Task[ + AgLiveCodeBenchXDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + # `float | str`: the report names its own score column and the language + # it measured, neither of which is a measurement. + dict[str, float | str], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + language: str | None = None, + container_lang: str | None = None, + k: int = 1, + n: int = 1, + max_concurrency: int = 4, + timeout: float = 15.0, + ): + """*language* is the string spliced into the prompt and has no default: + this benchmark measures a different thing per language, so picking one + silently would let two runs of the same task name mean different things. + + *container_lang* names the Agnostics verifier the evaluator should run; + when omitted it is ``language``'s first word, lowercased. *timeout* is + the container budget in seconds, sent as upstream sends it -- one number + serving as both the container's own ``timeout_s`` and the wall the + evaluator holds the process to. + """ + super().__init__(dataset=dataset, model=model, name=name) + if k > n: + raise ValueError( + f"pass@{k} needs at least {k} sample(s) per problem, got n={n}." + ) + if not language or not language.strip(): + raise ValueError( + "Ag-LiveCodeBench-X needs an explicit `language` (the string " + "spliced into the prompt, e.g. 'Lua' or 'Julia 1.10'); it is " + "the measurement, not a default. Set `container_lang` too when " + "the verifier tag is not `language`'s first word lowercased." + ) + self._language = language + self._container_lang = container_lang or language.split()[0].lower() + self._k = k + self._n = n + self._timeout = timeout + self._code_eval_api = os.getenv( + "SIEVAL_CODE_EVAL_API", "http://localhost:11451/evaluations" + ) + self._http_client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=max_concurrency) + ) + + @override + async def preprocess(self, raw, ctx): + system, user = _render_prompt(self._language, raw["question_content"]) + return build_prompt_record( + [system, user], + # No `reference`: the ground truth is a test suite, not a value. + extra={"question_id": raw["question_id"], "language": self._language}, + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + predictions: list[str | None] = [] + for choice in inf.texts: + fields = _parse_chat_adapter_fields(choice) + code = ( + None + if fields is None + else _extract_code_from_markdown(fields["solution"]) + ) + # Empty normalizes to None so `extracted` reports it as a miss, the + # same rule the LiveCodeBench sibling follows. It is still sent to + # the evaluator as "" below, which returns a real verdict rather + # than skipping the rollout. + predictions.append(code or None) + return build_prediction_record(predictions) + + @override + async def feedback(self, post, ctx): + cases = _decode_private_test_cases(ctx.raw_sample["private_test_cases"]) + inputs = [case["input"] for case in cases] + outputs = [case["output"] for case in cases] + + rollouts: list[RolloutJudgement] = [] + for rollout in post["rollouts"]: + idx = rollout["index"] + try: + resp = await self._http_client.post( + self._code_eval_api, + json={ + "uuid": f"{idx}-{time.perf_counter_ns()}", + "source": "agnostics", + "code": rollout.get("prediction") or "", + "lang": self._container_lang, + "test": {"inputs": inputs, "outputs": outputs}, + "timeout": self._timeout, + }, + # Beyond the container wall, for network latency and for the + # test suite itself: a decoded suite runs to tens of MB, and + # writing it is inside this deadline. + timeout=self._timeout + 300.0, + ) + resp.raise_for_status() + res = resp.json() + # should raise error if no `status` & `msg` field + correct, msg = res["status"], res["msg"] + data = res["data"] or {} + except Exception as e: + logger.warning( + "Evaluation error for sample {}: [{}] {}", + idx, + type(e).__name__, + e, + ) + raise e + + rollouts.append( + build_rollout_judgement( + idx, + correct, + extra={ + # The Agnostics `result` verbatim ("success", + # "fail:wrong-output", "fail:timeout", ...) or an + # `infra:` code. Stored raw; the taxonomy is the + # evaluator's to name, not this client's to guess. + "msg": msg, + "n_cases": data.get("n_cases"), + "n_passed": data.get("n_passed"), + "resources": { + key: value + for key, value in data.items() + if key not in ("n_cases", "n_passed") + }, + }, + ) + ) + + return True, build_judgement_record( + None, # the reference is the test suite below, not a value + rollouts, + extra={ + "question_id": ctx.raw_sample["question_id"], + "n_test_cases": len(cases), + # Every case is private and every problem is stdio: this release + # carries no public split and no `fn_name`, so there is nothing + # to report per problem beyond the count. + "io_mode": "stdio", + "container_lang": self._container_lang, + }, + ) + + @override + async def report(self, finals, fails): + total = len(finals) + len(fails) + # Upstream's `run_error_rate` counterpart: rollouts whose verdict came + # from the harness failing to run the container rather than from the + # submitted program. Kept as a count, not a rate, so it reads correctly + # against whatever denominator the consumer picks. + n_run_errors = sum( + 1 + for f in finals + for r in f.feedback_result["rollouts"] + if (r["extra"].get("msg") or "").startswith(_INFRA_PREFIX) + ) + # `votes=False`: two correct programs are not one answer, so there is + # nothing well-defined to take a majority over (RFC #74). + rolled = sampling_report( + finals, n=self._n, k=self._k, denominator=total, votes=False + ) + pass_at_1 = rolled["pass@1"] + metrics: dict[str, float | str] = { + "score": pass_at_1, + "fails": len(fails), + "n_run_errors": n_run_errors, + "pass@1": pass_at_1, + # Which language this column measured. Not a metric, but the column + # is meaningless without it and the task name cannot carry it. + "language": self._language, + "container_lang": self._container_lang, + SCORE_KEY_FIELD: "pass@1", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + if self._n > 1: + # At n=1 the rest only restates `pass@1`. + metrics.update(rolled) + # Outside the gate: extraction health is a fact about the parser, not + # about the draw, and it is what bounds the JSONAdapter divergence. + return metrics | health_metrics(finals) + + @override + async def shutdown(self): + await self._http_client.aclose() diff --git a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py new file mode 100644 index 00000000..0a25fb33 --- /dev/null +++ b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py @@ -0,0 +1,450 @@ +"""Unit tests for the Ag-LiveCodeBench-X 0-shot chat task. + +Three things are pinned here, all of which a reasonable-looking edit would break +silently: + +* **The prompt bytes.** They are a frozen ``dspy==3.0.0b2`` ``ChatAdapter`` + rendering, so nothing in the repo derives them and nothing but a hash can tell + you they still match. The two expectations below that look like typos -- an + 8-space indent inside the system message, a trailing space on one field line + and not the next -- are the rendering, not this file's mistakes. +* **The response parser**, whose expected values were captured by running + ``ChatAdapter.parse`` under that same pin. The indented-header case encodes an + upstream offset bug; "fixing" it would move scores. +* **The code extractor**, whose two surprising branches (no fence at all means + the whole reply is code; an unterminated fence means nothing is) are what a + cleanup usually gets wrong. + +The verifier half lives in the vendored evaluator, a separate deployable with its +own repo, so only what goes on the wire is asserted here. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import base64 +import hashlib +import json +import pickle +import zlib + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import Request, Response +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + TaskContext, + build_judgement_record, + build_prediction_record, + build_rollout_judgement, +) +from sieval.datasets.ag_livecodebench_x import AgLiveCodeBenchXDataset +from sieval.tasks.ag_livecodebench_x_0shot_gen import ( + AgLiveCodeBenchXZeroShotGenTask, + _extract_code_from_markdown, + _parse_chat_adapter_fields, + _render_prompt, +) +from tests.conftest import HandlerTransport + +# sha256 of `system + NUL + user` for the placeholder inputs below, captured from +# `ChatAdapter().format(ChainOfThought(SolveProblem).predict.signature, [], ...)` +# under upstream's pinned resolution (dspy==3.0.0b2, exclude-newer 2025-08-05). +_PROMPT_DIGEST = "69180174fafdd13d7523628d3d1df3ff9c2ef6e026af5c2529f696744ee3ae22" + +_CASES = [{"input": "2\n", "output": "4"}, {"input": "5\n", "output": "10"}] + + +def _encode(cases: list[dict]) -> str: + """LiveCodeBench's private-test encoding: JSON -> pickle -> zlib -> base64.""" + return base64.b64encode(zlib.compress(pickle.dumps(json.dumps(cases)))).decode() + + +def _raw() -> dict: + return { + "question_id": "1873_A", + "question_content": "Read an integer and print twice its value.", + "private_test_cases": _encode(_CASES), + } + + +class _StubChatModel(ChatModel): + def __init__(self): + super().__init__(model="mock-chat", api_key="fake") + + def _build_default_transport(self) -> HandlerTransport: + return HandlerTransport(self._stub_arun, "openai_chat") + + async def _stub_arun(self, req: Request) -> Response: + return Response(texts=("[[ ## solution ## ]]\nx",) * req.sampling.n) + + +class _Response: + """Enough of an httpx response for `feedback` -- the submission passed.""" + + @staticmethod + def raise_for_status() -> None: + return None + + @staticmethod + def json() -> dict: + return { + "status": True, + "msg": "success", + "data": {"n_cases": 1, "n_passed": 1}, + } + + +class _CapturingEvaluator: + """Stands in for the code-eval service, recording what was asked of it.""" + + def __init__(self): + self.bodies: list[dict] = [] + self.deadlines: list[float] = [] + + async def post(self, url, *, json, timeout): + _ = url + self.bodies.append(json) + self.deadlines.append(timeout) + return _Response() + + async def aclose(self) -> None: + return None + + +def _task(**kwargs) -> AgLiveCodeBenchXZeroShotGenTask: + kwargs.setdefault("language", "Lua") + dataset = AgLiveCodeBenchXDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([_raw()])}) + ) + return AgLiveCodeBenchXZeroShotGenTask(dataset, _StubChatModel(), **kwargs) + + +async def _post_one(prediction: str | None = "print(1)", **kwargs): + """Run `feedback` for one rollout and hand back what the evaluator saw.""" + task = _task(**kwargs) + await task._http_client.aclose() # the real client is never used + evaluator = _CapturingEvaluator() + task._http_client = evaluator + try: + await task.feedback( + build_prediction_record([prediction]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + return evaluator + + +# --------------------------------------------------------------------------- # +# The frozen prompt +# --------------------------------------------------------------------------- # +def test_prompt_bytes_match_the_captured_dspy_rendering(): + system, user = _render_prompt("<>", "<>") + assert system["role"] == "system" + assert user["role"] == "user" + + digest = hashlib.sha256( + (system["content"] + "\x00" + user["content"]).encode() + ).hexdigest() + # A mismatch means the constants were edited, not that DSPy changed: the + # rendering is frozen on purpose and there is no runtime DSPy here. + assert digest == _PROMPT_DIGEST + + +def test_instructions_keep_the_source_docstrings_indentation(): + # DSPy re-inserts the signature docstring into the system message with the + # Python source indentation intact, while `Signature.instructions` itself is + # dedented. Only observable by running it, so it is asserted rather than + # trusted. + system, _ = _render_prompt("Lua", "p") + assert ( + "objective is: \n Solve the following programming problem" + in (system["content"]) + ) + + +def test_the_field_list_keeps_its_trailing_space_asymmetry(): + # The list is built with a trailing ": " per line and stripped as a block, so + # the first line keeps its trailing space and the last one loses it. Looks + # like a typo; is the wire format. + system, _ = _render_prompt("Lua", "p") + assert "1. `programming_language` (str): \n" in system["content"] + assert "2. `problem_statement` (str):\n" in system["content"] + assert "1. `reasoning` (str): \n" in system["content"] + + +def test_the_problem_statement_may_contain_braces(): + # The system message carries literal `{programming_language}` placeholders as + # wire text, so the user turn is concatenated, never `format`ted. + _, user = _render_prompt("Lua", "print {x} and {{y}}") + assert "print {x} and {{y}}" in user["content"] + + +# --------------------------------------------------------------------------- # +# ChatAdapter response parsing +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + ("completion", "expected"), + [ + pytest.param( + "[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\nprint(1)\n\n" + "[[ ## completed ## ]]\n", + {"reasoning": "r", "solution": "print(1)"}, + id="canonical", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\nprint(1)", + {"reasoning": "r", "solution": "print(1)"}, + id="no-completed-marker", + ), + pytest.param( + "[[ ## solution ## ]]\nprint(1)\n\n[[ ## completed ## ]]\n", + None, + id="missing-reasoning-is-a-parse-failure", + ), + pytest.param( + "[[ ## reasoning ## ]] r\n[[ ## solution ## ]] print(1)\n", + {"reasoning": "r", "solution": "print(1)"}, + id="content-inline-with-the-header", + ), + pytest.param( + " [[ ## reasoning ## ]]\nr\n [[ ## solution ## ]]\nprint(1)\n", + {"reasoning": "]]\nr", "solution": "]]\nprint(1)"}, + id="indented-header-leaks-the-closing-brackets", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n[[ ## solution ## ]]\nfirst\n" + "[[ ## solution ## ]]\nsecond\n", + {"reasoning": "r", "solution": "first"}, + id="first-occurrence-wins", + ), + pytest.param( + "Sure!\n\n[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\nprint(1)\n", + {"reasoning": "r", "solution": "print(1)"}, + id="prose-before-the-first-header-is-dropped", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n[[ ## thoughts ## ]]\nx\n" + "[[ ## solution ## ]]\nprint(1)\n", + {"reasoning": "r", "solution": "print(1)"}, + id="unknown-headers-are-dropped", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\n\n" + "[[ ## completed ## ]]\n", + {"reasoning": "r", "solution": ""}, + id="blank-solution-parses-as-empty-not-missing", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr and then the tokens ran ou", + None, + id="truncated-before-the-solution-marker", + ), + ], +) +def test_parse_reproduces_chat_adapter(completion, expected): + # Every expectation here was produced by running dspy==3.0.0b2's own + # ChatAdapter.parse on the same input. + assert _parse_chat_adapter_fields(completion) == expected + + +# --------------------------------------------------------------------------- # +# Code extraction +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + ("markdown", "expected"), + [ + pytest.param("```lua\nprint(1)\n```", "print(1)", id="tagged-fence"), + pytest.param("```\nprint(1)\n```", "print(1)", id="untagged-fence"), + pytest.param( + "```lua\nfirst\n```\ntext\n```lua\nsecond\n```", + "first", + id="first-block-wins", + ), + pytest.param("print(1)", "print(1)", id="no-fence-is-all-code"), + pytest.param("```lua\nprint(1)", None, id="unterminated-fence-is-nothing"), + pytest.param( + "```python\nsolve()\n# Example usage:\nsolve(1)\n```", + "solve()", + id="example-usage-truncation", + ), + pytest.param("", "", id="empty-in-empty-out"), + pytest.param(None, None, id="none-in-none-out"), + ], +) +def test_extract_code_reproduces_upstream(markdown, expected): + assert _extract_code_from_markdown(markdown) == expected + + +# --------------------------------------------------------------------------- # +# language / container_lang +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("language", [None, "", " "]) +def test_language_has_no_default(language): + # The benchmark measures a different thing per language, so a silent default + # would let two runs of one task name mean different things. + with pytest.raises(ValueError, match="explicit `language`"): + _task(language=language) + + +@pytest.mark.parametrize( + ("language", "expected_tag"), + [ + ("Lua", "lua"), + ("Lua /nothink", "lua"), + ("Julia 1.10", "julia"), + ("OCaml", "ocaml"), + ], +) +def test_container_tag_is_derived_from_the_first_word(language, expected_tag): + assert _task(language=language)._container_lang == expected_tag + + +def test_container_tag_can_be_overridden(): + # Upstream warns the two are independent; "C++" is the obvious case where the + # derivation cannot be right. + task = _task(language="C++", container_lang="cpp") + assert task._container_lang == "cpp" + + +@pytest.mark.anyio +async def test_the_prompt_language_reaches_the_prompt_not_the_container(): + task = _task(language="Lua /nothink", container_lang="lua") + try: + record = await task.preprocess( + _raw(), TaskContext(sample_id=0, raw_sample=_raw()) + ) + finally: + await task.shutdown() + + user = record["prompt"][1]["content"] + assert "Lua /nothink" in user + assert record["extra"]["language"] == "Lua /nothink" + + +# --------------------------------------------------------------------------- # +# What goes on the wire +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_the_request_carries_the_agnostics_source_and_suite(): + evaluator = await _post_one() + + (body,) = evaluator.bodies + assert body["source"] == "agnostics" + assert body["lang"] == "lua" + assert body["code"] == "print(1)" + # Decoded from the sample's own base64/zlib/pickle column, split into the + # parallel lists the evaluator's request model already speaks. + assert body["test"] == {"inputs": ["2\n", "5\n"], "outputs": ["4", "10"]} + # Every problem in this release is stdio, so no `fn_name` is ever sent. + assert "fn_name" not in body["test"] + + +@pytest.mark.anyio +async def test_the_default_timeout_is_upstreams_fifteen_seconds(): + evaluator = await _post_one() + + (body,) = evaluator.bodies + assert body["timeout"] == 15.0 + # The HTTP deadline must sit outside the container wall, or a timing-out + # submission surfaces as a transport error instead of a verdict. The margin + # is large because writing a decoded suite (tens of MB) is inside it. + assert evaluator.deadlines == [15.0 + 300.0] + + +@pytest.mark.anyio +async def test_an_unextractable_answer_is_still_graded(): + # `None` on disk means "could not extract"; on the wire it is "", so the + # verifier returns a real verdict rather than the rollout being skipped. + evaluator = await _post_one(prediction=None) + + (body,) = evaluator.bodies + assert body["code"] == "" + + +def test_k_may_not_exceed_n(): + with pytest.raises(ValueError, match="pass@2"): + _task(k=2, n=1) + + +# --------------------------------------------------------------------------- # +# report() +# --------------------------------------------------------------------------- # +async def _report(msgs: list[str], corrects: list[bool] | None = None): + task = _task() + corrects = corrects if corrects is not None else [False] * len(msgs) + try: + finals = [ + TaskContext( + sample_id=i, + raw_sample=_raw(), + feedback_result=build_judgement_record( + None, + [build_rollout_judgement(0, ok, extra={"msg": msg})], + ), + ) + for i, (msg, ok) in enumerate(zip(msgs, corrects, strict=True)) + ] + return await task.report(finals, []) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_infra_failures_are_counted_separately_from_wrong_answers(): + # Upstream's `run_error_rate` counterpart. It must not touch pass@1: an + # unrunnable container is still a rollout that did not pass. + report = await _report( + ["success", "fail:wrong-output", "infra:timeout", "infra:exit 125: no image"], + corrects=[True, False, False, False], + ) + + assert report["n_run_errors"] == 2 + assert report["pass@1"] == pytest.approx(25.0) + assert report["score"] == report["pass@1"] + + +@pytest.mark.anyio +async def test_a_null_message_does_not_crash_the_counter(): + # An absent `msg` is `None` on disk after serialization drops it. + report = await _report([None]) # type: ignore[list-item] + + assert report["n_run_errors"] == 0 + + +@pytest.mark.anyio +async def test_the_report_declares_its_column_and_the_language_it_measured(): + report = await _report(["success"], corrects=[True]) + + assert report["score_key"] == "pass@1" + assert report["denominator_policy"] == "requested" + # The task name cannot carry the language, so the report has to. + assert report["language"] == "Lua" + assert report["container_lang"] == "lua" + + +@pytest.mark.anyio +async def test_a_pipeline_failure_counts_against_the_score(): + # `DENOMINATOR_REQUESTED`, matching upstream's `pass1`: it divides by rows + # written, and a generation that produced no solution is still one of them. + task = _task() + try: + finals = [ + TaskContext( + sample_id=0, + raw_sample=_raw(), + feedback_result=build_judgement_record( + None, [build_rollout_judgement(0, True, extra={"msg": "success"})] + ), + ) + ] + report = await task.report( + finals, [TaskContext(sample_id=1, raw_sample=_raw())] + ) + finally: + await task.shutdown() + + assert report["fails"] == 1 + assert report["pass@1"] == pytest.approx(50.0) diff --git a/vendor/code-evaluator/README.md b/vendor/code-evaluator/README.md index ddb6ba22..49e856a9 100644 --- a/vendor/code-evaluator/README.md +++ b/vendor/code-evaluator/README.md @@ -13,6 +13,8 @@ code either by running it directly or by checking it against test cases. - SciCode (python only) - NL2SH-ALFA / InterCode-ALFA (Bash) — **separate route, separate images** - QuoteBench (bash only; fixture + final-state grading, see below) +- Ag-LiveCodeBench-X and anything else speaking the Agnostics protocol (any + language, via a per-language verifier container) HumanEval and SciCode run the submitted code directly; LiveCodeBench needs a function name and compares inputs against expected outputs. A SciCode program @@ -25,6 +27,12 @@ baseline tree with a reset in between. That tree comes from the image, which is upstream's, so there are five of them — one per prepared filesystem — and one service instance hosts exactly one. See "Shell endpoint" below. +`source: "agnostics"` is odd in a different way: it keeps this route's shape but +nothing runs in this process. The request is handed to a container that owns the +language toolchain, which is what makes Lua / R / Julia / OCaml / Fortran +reachable without installing five toolchains here. See "Agnostics protocol" +below. + ## Setup ### Python environment @@ -139,12 +147,14 @@ Fields: - `uuid` - `source`: `"human-eval"` | `"mbpp"` | `"livecodebench"` | `"scicode"` | - `"quotebench"` + `"quotebench"` | `"agnostics"` - `lang`: `python` | `javascript` | `typescript` | `cpp` | `bash` | `perl` — ask `GET /languages` rather than hard-coding this list; ignored by - `quotebench`, which is Bash-only + `quotebench`, which is Bash-only; for `"agnostics"` it is instead + the **container tag** (`lua`, `julia`, `r`, `ocaml`, `fortran`, …) - `code`: the code, as a string — for `quotebench`, the model's reply -- `test`: LiveCodeBench-specific test description (`fn_name` / `inputs` / `outputs`) +- `test`: LiveCodeBench-specific test description (`fn_name` / `inputs` / `outputs`). + `"agnostics"` requires it and uses only `inputs` / `outputs` - `timeout`: float (optional, seconds; defaults below) — a wall for the **whole suite** - `timeout_per_case`: float (optional, seconds) — budget each test case individually, as official LiveCodeBench does @@ -294,6 +304,46 @@ curl -X POST http://localhost:11451/evaluations \ -d '{"uuid":"demo","source":"human-eval","lang":"python","code":"print(42)","memory_limit":1024}' ``` +## Agnostics protocol + +`source: "agnostics"` forwards the submission to a verifier container over one +JSON line in, one JSON line out: + +```text +in { "code": str, "timeout_s": int, + "test_cases": [ { "input": str, "output": str }, ... ] } +out { "result": "success" } + | { "result": "fail:wrong-output", "expected": str, "got": str, "stderr": str } + | { "result": "fail:error", "exit_code": int, "stdout": str, "stderr": str } + | { "result": "fail:timeout", "stdout": str, "stderr": str } + | { "result": "fail:other", "stdout": str, "stderr": str } +``` + +Only `"success"` is a pass. Other result codes and extra fields are allowed by +the protocol, so anything unrecognized is reported as a failure, not an error. +The response `msg` is the container's `result` verbatim, or an `infra:` +code (`spawn` / `stdin` / `timeout` / `exit ` / `decode` / `bad-lang` / +`no-test`) when the container could not be asked — so a caller can tell "the +program is wrong" from "the harness could not run it". + +The verdict covers the whole suite, so `n_cases` / `n_passed` are the +all-or-nothing pair (`1/1` or `1/0`); the suite's real size is on the request. + +The command is **deployment configuration, not a request field** — a caller able +to name the image could run any container on this host. It defaults to upstream's +own invocation, with `{lang}` filled from the request: + +```bash +podman run --rm -i --tmpfs /ramdisk:size=512m,exec ghcr.io/nuprl/agnostics:{lang} +``` + +Override the whole command with `CODE_EVAL_AGNOSTICS_COMMAND` for a different +runtime (upstream also supports `apptainer run --contain --writable-tmpfs .sif`), +a mirrored registry, or a local test double. `lang` must match +`[a-z0-9][a-z0-9_.+-]{0,31}` since it lands in an argv slot. `memory_limit` and +`test.fn_name` are ignored: the container owns its own limits, and the protocol +has no call-based mode. + ## Resource limits and defaults ### Timeout @@ -309,6 +359,12 @@ When a request omits `timeout`, these defaults apply: cache; charging it nothing would let a pathological one hang. - livecodebench: 6s + 2s * number of cases, or `(timeout_per_case + 1) * n + 5` when `timeout_per_case` was sent — upstream's own backstop shape +- agnostics: 15s, the value upstream's README uses. It serves as **both** the + container's own `timeout_s` and the wall the process is held to, as upstream + sends it — so a suite whose per-case budgets sum past the wall is killed from + outside before it can report its own `fail:timeout`. Writing the payload gets + its own 300s budget, since a decoded suite runs to tens of MB and a slow write + is not a slow program. For the table-driven languages (cpp / bash / perl) the wall is enforced against the whole **process group**: the program is spawned in its own session and a @@ -383,6 +439,10 @@ whole protocol against a throwaway git repo with no container. - `tests/`: the shell backend's tests (no container required) - `README.md`: this document +The `agnostics` source needs no image file or requirements of its own — the +container it delegates to supplies the toolchain, and only a container runtime +(`podman`, or `apptainer` for a `.sif`) has to exist on the host. + ## Note Isolation is not hardened. Do not run untrusted or high-risk code. diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index d519380f..27073c58 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -381,3 +381,48 @@ Two kinds, and the difference is a decision rather than a status: same 224/224 inside it as on the host, which is what promoted the sieval tasks to `stable`. `docker build` itself stays unexercised; the reproduction recipe is in `tests/acceptance/quotebench/README.md`. +- `app/exec_agnostics.py` (new), `app/server.py`, `README.md` — **the Agnostics + protocol** as `source="agnostics"`, which is what makes a non-Python language + reachable at all. Nothing executes in this process: the request is forwarded to + a per-language verifier container over one JSON line in, one JSON line out + (`{code, timeout_s, test_cases}` -> `{result: "success" | "fail:*", ...}`), and + only `"success"` is a pass. Upstream of the *protocol* is + nuprl/Ag-LiveCodeBench-X at `b7b273ef`; sieval's client is + `ag_livecodebench_x_0shot_gen`. + + Three deliberate choices, each of which a reviewer will want to push back on: + + * **The container command is deployment config, not a request field.** Upstream + passes `--container-name` on its own CLI, which is safe when the harness and + the caller are the same process. Here they are not, so a client able to name + the image could run an arbitrary container on the evaluator host. `lang` is + all the client sends, constrained to `[a-z0-9][a-z0-9_.+-]{0,31}` because it + lands in an argv slot, and the command comes from + `CODE_EVAL_AGNOSTICS_COMMAND`, defaulting to upstream's own podman + invocation. That is the one place this deviates from upstream's shape rather + than its behaviour. + * **`infra:` is reported instead of upstream's collapse to `"fail"`.** + Upstream turns every harness-side failure (non-zero exit, undecodable stdout) + into `result: "fail"` and recovers only the stdin-write case, by matching a + stderr suffix. The split is named here, where it is known, rather than left to + a client-side classifier over free text. **It does not change what counts as a + pass** -- only `"success"` does, either way -- so `pass@1` is unaffected and + only the diagnostic count differs (sieval's `n_run_errors` is therefore + broader than upstream's `run_error_rate` numerator). + * **One number in two roles, kept.** `timeout` is sent as both the container's + `timeout_s` and the wall the process is held to, because upstream does that + and widening the wall would move scores. Consequence, confirmed by running it: + the outer wall is armed first, so the container's own `fail:timeout` is + effectively unreachable and a timing-out submission surfaces as + `infra:timeout`. Writing the payload keeps upstream's separate 300s budget + (`stdin_write_timeout`), since a decoded LiveCodeBench suite is tens of MB. + + Resource stats are the podman *client* process's, not the container's -- the + existing `monitor_process_resources` watches the pid it spawned. Reported anyway + so `data` is never null, but do not read them as the submission's cost. + + Verified against a local stub verifier speaking the protocol (no podman on the + dev box): `success` / `fail:wrong-output` / `fail:error` pass through verbatim, + and `infra:timeout` / `infra:bad-lang` / `infra:no-test` all fire. Not yet + upstream -- land in `scitix/code-evaluator` and re-vendor; tests belong there + rather than under `tests/`, which mirrors `sieval/`. diff --git a/vendor/code-evaluator/app/exec_agnostics.py b/vendor/code-evaluator/app/exec_agnostics.py new file mode 100644 index 00000000..a41827cd --- /dev/null +++ b/vendor/code-evaluator/app/exec_agnostics.py @@ -0,0 +1,152 @@ +"""Run a submission through an Agnostics verifier container. + +The Agnostics protocol (nuprl/Ag-LiveCodeBench-X) is one JSON line in, one JSON +line out: + + in { "code": str, "timeout_s": int, + "test_cases": [ { "input": str, "output": str }, ... ] } + out { "result": "success" } + | { "result": "fail:wrong-output", "expected": str, "got": str, ... } + | { "result": "fail:error", "exit_code": int, ... } + | { "result": "fail:timeout", ... } + | { "result": "fail:other", ... } + +Only ``"success"`` counts as a pass; the protocol explicitly allows other result +codes and extra fields, so anything unrecognized is a failure rather than an +error. + +The container command is **deployment configuration**, never a request field: a +caller able to name the image could run an arbitrary container on this host. The +default is upstream's own invocation, with ``{lang}`` filled from the request's +``lang``. +""" + +import asyncio +import json +import os +import re +import shlex + +from .resource_monitor import ResourceStats, monitor_process_resources + +_COMMAND_ENV_VAR = "CODE_EVAL_AGNOSTICS_COMMAND" + +# Upstream's podman invocation, verbatim apart from the `{lang}` placeholder. +# Override the whole command via the env var above for a different runtime +# (upstream also supports `apptainer run --contain --writable-tmpfs .sif`), +# a mirrored registry, or a local test double. +_DEFAULT_COMMAND = ( + "podman run --rm -i --tmpfs /ramdisk:size=512m,exec " + "ghcr.io/nuprl/agnostics:{lang}" +) + +# `lang` reaches an argv slot, so it is constrained to what a container tag can +# be. Not a sandbox -- the command template is trusted and this is not -- but it +# keeps a tag from smuggling in a separator or a path. +_LANG_PATTERN = re.compile(r"\A[a-z0-9][a-z0-9_.+-]{0,31}\Z") + +# Upstream gives writing the payload its own budget, separate from the wall the +# process runs under, because a decoded LiveCodeBench suite is tens of MB and a +# slow write is not a slow program. Its `stdin_write_timeout=300` is kept. +_STDIN_WRITE_TIMEOUT = 300.0 + + +def verifier_command(lang: str) -> list[str]: + template = os.environ.get(_COMMAND_ENV_VAR) or _DEFAULT_COMMAND + return [part.replace("{lang}", lang) for part in shlex.split(template)] + + +async def execute_agnostics( + code: str, + inputs: list[str], + expect_outputs: list[str], + lang: str, + timeout: float, +) -> tuple[bool, str, ResourceStats]: + """Return ``(passed, msg, stats)``. + + ``msg`` is the container's ``result`` value verbatim when the container ran + and spoke the protocol, and an ``infra:`` code when it did not, so a + client can separate "the program is wrong" from "the harness could not ask". + Upstream collapses every one of the latter to ``"fail"`` and recovers only + the stdin-write case, from a stderr suffix; the split is named here instead, + where it is known. It does not change what counts as a pass: only + ``"success"`` does, either way. + + ``timeout`` serves as both the container's own ``timeout_s`` and the wall + this process is held to -- one number in both roles, as upstream sends it. + A suite whose per-case budgets can sum past the wall is therefore killed + from outside before it can report its own ``fail:timeout``; that tension is + upstream's, and widening the wall here would move scores. + """ + stats = ResourceStats() + if not _LANG_PATTERN.match(lang): + return False, f"infra:bad-lang: {lang!r}", stats + + payload = json.dumps( + { + "code": code, + # The protocol types it as an int, and upstream passes its + # `--timeout-seconds` straight through. + "timeout_s": int(timeout), + "test_cases": [ + {"input": stdin, "output": stdout} + for stdin, stdout in zip(inputs, expect_outputs, strict=True) + ], + } + ).encode("utf-8") + + try: + proc = await asyncio.create_subprocess_exec( + *verifier_command(lang), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except Exception as e: + return False, f"infra:spawn: [{type(e).__name__}] {e}", stats + + if proc.pid is not None: + stats, stop_event = await monitor_process_resources(proc.pid) + else: + stop_event = asyncio.Event() + stop_event.set() + + try: + assert proc.stdin is not None + try: + proc.stdin.write(payload) + await asyncio.wait_for(proc.stdin.drain(), timeout=_STDIN_WRITE_TIMEOUT) + proc.stdin.close() + except (asyncio.TimeoutError, ConnectionError, OSError) as e: + proc.kill() + await proc.wait() + return False, f"infra:stdin: [{type(e).__name__}] {e}", stats + + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + return False, "infra:timeout", stats + + if proc.returncode != 0: + detail = stderr.decode(errors="replace").strip()[-500:] + return False, f"infra:exit {proc.returncode}: {detail}", stats + + try: + verdict = json.loads(stdout) + except json.JSONDecodeError: + return False, "infra:decode", stats + if not isinstance(verdict, dict): + return False, "infra:decode", stats + + result = verdict.get("result") + if not isinstance(result, str): + return False, "infra:decode", stats + return result == "success", result, stats + finally: + stop_event.set() + await asyncio.sleep(0.1) # give the monitor time to finish diff --git a/vendor/code-evaluator/app/server.py b/vendor/code-evaluator/app/server.py index b94204ef..2aa25acb 100644 --- a/vendor/code-evaluator/app/server.py +++ b/vendor/code-evaluator/app/server.py @@ -9,6 +9,7 @@ from loguru import logger from pydantic import BaseModel +from .exec_agnostics import execute_agnostics from .exec_js import execute_code as exec_js from .exec_lang import LANGUAGES, toolchain_entry, toolchain_present from .exec_lang import execute_code as exec_lang @@ -482,6 +483,59 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: scenarios_digest=scenarios_digest(), ), ) + elif sample.source == "agnostics": + # 'agnostics': the Agnostics protocol (nuprl/Ag-LiveCodeBench-X) -- one + # JSON line to a per-language verifier container, one JSON line back. + # `lang` names the container, not an executor in this process, which is + # what makes Lua / R / Julia / OCaml / Fortran reachable at all. + logger.debug(f"code to exec:\n{sample.code}") + + if sample.test is None: + # Unlike 'livecodebench', there is no direct-run reading: the + # protocol has no shape that carries a submission with no suite. + msg = "infra:no-test: source 'agnostics' requires a test suite" + logger.error(msg) + return BasicResponse(status=False, msg=msg, data=None) + + # Upstream's `--timeout-seconds` is required and its README uses 15. + timeout = sample.timeout if sample.timeout is not None else 15.0 + # `test.fn_name` is ignored on purpose: this set is entirely + # stdin/stdout, and the protocol has no call-based mode to route it to. + # `memory_limit` likewise -- the container owns its own limits, and they + # belong in the command template rather than in a request field. + ok, msg, stats = await execute_agnostics( + code=sample.code, + inputs=sample.test.inputs, + expect_outputs=sample.test.outputs, + lang=sample.lang, + timeout=timeout, + ) + + logger.info( + f"evaluate sample '{sample.uuid}' from '{sample.source}', " + f"language: {sample.lang}, timeout: {timeout}, " + f"cases: {len(sample.test.inputs)}, kwargs: {sample.kwargs}, " + f"status: {ok}, msg: {msg}, " + f"avg_cpu: {stats.cpu_percent:.2f}%, " + f"peak_cpu: {stats.peak_cpu_percent:.2f}%, " + f"avg_memory: {stats.memory_mb:.2f}MB, " + f"peak_memory: {stats.peak_memory_mb:.2f}MB" + ) + return BasicResponse( + status=ok, + msg=msg, + data=ResourceMetrics( + avg_cpu_percent=stats.cpu_percent, + peak_cpu_percent=stats.peak_cpu_percent, + avg_memory_mb=stats.memory_mb, + peak_memory_mb=stats.peak_memory_mb, + # The verifier returns one verdict for the whole suite, so this + # is the all-or-nothing pair the direct-run modes report. The + # suite's real size is on the request, not in this count. + n_cases=1, + n_passed=int(ok), + ), + ) else: logger.error(f"not supported data source: {sample.source}") return BasicResponse( From b3fb53dd4963e92458e59d6adb91ac8ae419e879 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sun, 23 Aug 2026 00:49:04 +0800 Subject: [PATCH 02/10] fix(tasks): Ag-LiveCodeBench-X verifier tags are file extensions, not names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ghcr.io/nuprl/agnostics` publishes exactly `lua, r, python, jl, java, cpp, ml, f90` — Julia is `jl`, OCaml is `ml`, Fortran is `f90`. Deriving the tag by lowercasing the language's first word, as this shipped, named no published image for three of the five languages the paper reports, so every rollout would have come back `infra:exit` (no such image) and the task would have scored a hard 0 for reasons that look like an infrastructure outage. `_CONTAINER_TAG_BY_LANGUAGE` maps the extension-tagged half and leaves the rest falling through to the lowercased first word, which is correct for `lua` / `r` / `python` / `java` and the only sensible guess for a container the table has never seen. `container_lang` still overrides it, which is what a private registry needs. Found by querying the registry's tag list rather than by assuming it mirrors the framework repo's directory names, which are spelled out in full (`executors/julia`, `executors/ocaml`, `executors/fortran`). Co-Authored-By: Claude Opus 5 (1M context) --- sieval/meta/index.json | 2 +- sieval/tasks/ag_livecodebench_x_0shot_gen.py | 51 ++++++++++++++++--- .../test_ag_livecodebench_x_0shot_gen.py | 31 ++++++++--- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index ff6ef81d..4887b772 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1487,7 +1487,7 @@ "reference_impl": { "source": "nuprl/Ag-LiveCodeBench-X", "url": "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py", - "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, same protocol and same image, different process boundary; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). No sieval run has been aligned against a published number yet, which is what status='experimental' records." + "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, same protocol and same image, different process boundary; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Verifier tags are upstream's own, which are FILE EXTENSIONS, not language names -- ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased language name would name no image. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). No sieval run has been aligned against a published number yet, which is what status='experimental' records." }, "status": "experimental", "reference_kind": "procedure" diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py index a1afc474..a19b6476 100644 --- a/sieval/tasks/ag_livecodebench_x_0shot_gen.py +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -13,10 +13,12 @@ ``"Julia 1.10"``, ``"Lua /nothink"`` — upstream's own README suggests appending a version or a thinking-disable directive there). * ``container_lang`` selects the verifier the evaluator runs. Upstream warns that - nothing checks the two against each other; the default here derives the tag - from ``language``'s first word, lowercased, which is a convenience and not a - guarantee. Override it whenever the derivation is wrong (``"C++"`` is not a - container tag). + nothing checks the two against each other, and its published tags are **file + extensions rather than language names** -- ``ghcr.io/nuprl/agnostics`` ships + exactly ``lua, r, python, jl, java, cpp, ml, f90``, so Julia is ``jl``, OCaml + is ``ml`` and Fortran is ``f90``. The default maps those through + ``_CONTAINER_TAG_BY_LANGUAGE`` and otherwise lowercases ``language``'s first + word; override it for a private registry or a hand-built container. Both are recorded in ``report()``, because the task name cannot hold them: one registered task measures a different quantity per language, and a leaderboard @@ -189,6 +191,36 @@ # this string here -- the task reads the response's `status`. _INFRA_PREFIX = "infra:" +# Upstream tags its published verifiers by **file extension**, not by language +# name: `ghcr.io/nuprl/agnostics` ships exactly +# `lua, r, python, jl, java, cpp, ml, f90`. So lowercasing the language's first +# word -- the obvious derivation, and the one this task shipped first -- names a +# tag that does not exist for three of the five languages the paper reports, and +# every rollout comes back `infra:exit` (no such image). The table is upstream's +# tag list, keyed by what a `language` string plausibly starts with; anything not +# in it falls through to the lowercased first word, which is right for `lua` / +# `r` / `python` / `java` and is the only sensible guess for a container the +# table has never seen. +_CONTAINER_TAG_BY_LANGUAGE = { + "julia": "jl", + "ocaml": "ml", + "fortran": "f90", + "c++": "cpp", + "cplusplus": "cpp", +} + + +def _derive_container_tag(language: str) -> str: + """Upstream's published verifier tag for a prompt-language string. + + Only a default -- ``container_lang`` overrides it, which is what a private + registry or a hand-built container needs. Reads the first word, since a + ``language`` may carry a version or a directive (``"Julia 1.10"``, + ``"Lua /nothink"``). + """ + first_word = language.split()[0].lower() + return _CONTAINER_TAG_BY_LANGUAGE.get(first_word, first_word) + def _render_prompt( language: str, question_content: str @@ -334,7 +366,11 @@ def _decode_private_test_cases(text: str) -> list[dict[str, str]]: "verifier container is run by sieval's code-evaluator rather than " "from the harness process, same protocol and same image, different " "process boundary; (3) upstream's pre-run 'Say this is a test!' " - "smoke call is not reproduced. Upstream defaults: n=1 completion, " + "smoke call is not reproduced. Verifier tags are upstream's own, " + "which are FILE EXTENSIONS, not language names -- " + "ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, " + "f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased " + "language name would name no image. Upstream defaults: n=1 completion, " "temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout " "supplied per run (README uses 15s, and sends that one number as " "BOTH the container's timeout_s and the outer process wall). No " @@ -372,7 +408,8 @@ def __init__( silently would let two runs of the same task name mean different things. *container_lang* names the Agnostics verifier the evaluator should run; - when omitted it is ``language``'s first word, lowercased. *timeout* is + when omitted it is derived by :func:`_derive_container_tag`, which knows + that upstream tags by file extension (Julia is ``jl``). *timeout* is the container budget in seconds, sent as upstream sends it -- one number serving as both the container's own ``timeout_s`` and the wall the evaluator holds the process to. @@ -390,7 +427,7 @@ def __init__( "the verifier tag is not `language`'s first word lowercased." ) self._language = language - self._container_lang = container_lang or language.split()[0].lower() + self._container_lang = container_lang or _derive_container_tag(language) self._k = k self._n = n self._timeout = timeout diff --git a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py index 0a25fb33..612593a9 100644 --- a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py +++ b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py @@ -292,21 +292,38 @@ def test_language_has_no_default(language): @pytest.mark.parametrize( ("language", "expected_tag"), [ + # Tags upstream publishes under the language's own name. ("Lua", "lua"), ("Lua /nothink", "lua"), - ("Julia 1.10", "julia"), - ("OCaml", "ocaml"), + ("R", "r"), + ("Python", "python"), + ("Java", "java"), + # ...and the ones tagged by FILE EXTENSION instead. Lowercasing the first + # word -- the obvious derivation -- names no published image for these, + # so every rollout would come back `infra:exit`. Three of the five + # languages the paper reports are in this half. + ("Julia 1.10", "jl"), + ("OCaml", "ml"), + ("Fortran", "f90"), + ("C++", "cpp"), ], ) -def test_container_tag_is_derived_from_the_first_word(language, expected_tag): +def test_container_tag_matches_upstreams_published_tags(language, expected_tag): + # ghcr.io/nuprl/agnostics ships exactly: lua, r, python, jl, java, cpp, ml, f90. assert _task(language=language)._container_lang == expected_tag +def test_an_unknown_language_falls_through_to_its_first_word(): + # No table entry is not an error: a private registry may publish any tag, and + # the first word is the only sensible guess. + assert _task(language="Zig 0.13")._container_lang == "zig" + + def test_container_tag_can_be_overridden(): - # Upstream warns the two are independent; "C++" is the obvious case where the - # derivation cannot be right. - task = _task(language="C++", container_lang="cpp") - assert task._container_lang == "cpp" + # What a private registry or a hand-built container needs; upstream warns the + # prompt language and the verifier tag are independent. + task = _task(language="Julia", container_lang="julia-1-10-custom") + assert task._container_lang == "julia-1-10-custom" @pytest.mark.anyio From 49f07b1ef1020fc58489831d7a51405bc7836e91 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sun, 23 Aug 2026 01:03:01 +0800 Subject: [PATCH 03/10] feat(vendor): pin the Agnostics verifier by digest, not by tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verifier decides scores, so it gets what a dataset revision gets. A tag can be moved under a finished leaderboard without anything on disk changing, which is exactly the drift `--resume` and pinned revisions exist to prevent everywhere else in this repo. `_IMAGE_DIGESTS` maps each of upstream's eight published tags to a digest resolved from the registry on 2026-08-23, each verified against the manifest's own `Docker-Content-Digest`. A language with no pinned digest is REFUSED (`infra:unpinned-lang`) rather than falling back to the floating tag: an unpinned verifier scores silently, which is the failure the table exists to prevent. All eight are single-platform linux/amd64 manifests, so the pin binds the architecture too — arm64 needs the command override. A pin nothing records is not a pin, so the resolved reference travels back as `data.verifier_image`, onto each rollout's judgement, and is surfaced once as `report()`'s `verifier_image`. Two verifiers in one run (an evaluator re-pointed mid-run) are both reported rather than one being picked to stand for the rest, and an unrecorded one is omitted rather than blanked — an empty string would read as "ran unpinned", a different claim. Under a command override the field is `None` unless the override's template actually contains `{image}`: naming a digest that did not run is worse than reporting nothing. Verified live on both paths: lua/jl/ml/f90 resolve and report their digests, `julia` and `rust` are refused, and a `{lang}`-templated override reports no image. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/meta/index.json | 2 +- sieval/tasks/ag_livecodebench_x_0shot_gen.py | 57 ++++++-- .../test_ag_livecodebench_x_0shot_gen.py | 58 ++++++++- vendor/code-evaluator/README.md | 27 +++- vendor/code-evaluator/VENDORED.md | 36 ++++- vendor/code-evaluator/app/exec_agnostics.py | 123 ++++++++++++++---- vendor/code-evaluator/app/server.py | 10 +- 7 files changed, 262 insertions(+), 51 deletions(-) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 4887b772..8c3f6ff9 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1487,7 +1487,7 @@ "reference_impl": { "source": "nuprl/Ag-LiveCodeBench-X", "url": "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py", - "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, same protocol and same image, different process boundary; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Verifier tags are upstream's own, which are FILE EXTENSIONS, not language names -- ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased language name would name no image. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). No sieval run has been aligned against a published number yet, which is what status='experimental' records." + "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, and pinned by DIGEST rather than by upstream's mutable tag -- the digest that scored each rollout comes back on the verdict and is surfaced as report()'s verifier_image; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Verifier tags are upstream's own, which are FILE EXTENSIONS, not language names -- ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased language name would name no image. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). No sieval run has been aligned against a published number yet, which is what status='experimental' records." }, "status": "experimental", "reference_kind": "procedure" diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py index a19b6476..744e0ca8 100644 --- a/sieval/tasks/ag_livecodebench_x_0shot_gen.py +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -62,13 +62,17 @@ row; upstream's own README warns that truncation at its default ``max_tokens=5000`` is common enough to mention, and truncation is precisely what strips the closing marker. -* **The container is run by the evaluator, not by this task.** Upstream shells - out to ``podman run ... ghcr.io/nuprl/agnostics:`` (or ``apptainer`` for - a ``.sif``) from the harness process. Here the task POSTs to sieval's - code-evaluator, which runs that same container server-side; the invocation is - the evaluator's deployment config, not a per-request field, so a client cannot - name an arbitrary image to execute. Same verifier, same protocol, different - process boundary. +* **The container is run by the evaluator, not by this task, and pinned by + digest.** Upstream shells out to ``podman run ... ghcr.io/nuprl/agnostics:`` + (or ``apptainer`` for a ``.sif``) from the harness process, against the mutable + tag. Here the task POSTs to sieval's code-evaluator, which runs that same + container server-side from a digest — the verifier is scoring data, so it is + pinned the way a dataset revision is, and a tag moved under a finished + leaderboard cannot change a score silently. The invocation is the evaluator's + deployment config, not a per-request field, so a client cannot name an + arbitrary image to execute; the digest that ran comes back on each verdict and + is surfaced once as ``report()``'s ``verifier_image``. Same verifier, same + protocol, stronger pin, different process boundary. * **No smoke call.** Upstream sends ``lm("Say this is a test!")`` before the run; it measures nothing and is not reproduced. * **No DSPy cache.** Upstream defaults it off (``--enable-dspy-cache`` is opt-in @@ -364,8 +368,10 @@ def _decode_private_test_cases(text: str) -> list[dict[str, str]]: "reply missing either marker scores 0 here and the gap is exactly " "n_unextracted, making this a lower bound; (2) the Agnostics " "verifier container is run by sieval's code-evaluator rather than " - "from the harness process, same protocol and same image, different " - "process boundary; (3) upstream's pre-run 'Say this is a test!' " + "from the harness process, and pinned by DIGEST rather than by " + "upstream's mutable tag -- the digest that scored each rollout comes " + "back on the verdict and is surfaced as report()'s verifier_image; " + "(3) upstream's pre-run 'Say this is a test!' " "smoke call is not reproduced. Verifier tags are upstream's own, " "which are FILE EXTENSIONS, not language names -- " "ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, " @@ -519,10 +525,16 @@ async def feedback(self, post, ctx): "msg": msg, "n_cases": data.get("n_cases"), "n_passed": data.get("n_passed"), + # Which pinned verifier produced this verdict. Recorded + # per rollout rather than once per run because it is the + # evaluator that resolves it, and a deployment could be + # re-pointed mid-run; `None` means the evaluator ran a + # command override, where the image is unknowable. + "verifier_image": data.get("verifier_image"), "resources": { key: value for key, value in data.items() - if key not in ("n_cases", "n_passed") + if key not in ("n_cases", "n_passed", "verifier_image") }, }, ) @@ -542,6 +554,25 @@ async def feedback(self, post, ctx): }, ) + @staticmethod + def _verifier_image(finals) -> str | None: + """The pinned verifier behind this column, or ``None`` if unrecorded. + + A run should see exactly one image. More than one means the evaluator was + re-pointed mid-run, which makes the column a mix of two graders -- so all + of them are reported, sorted and joined, rather than one being picked to + stand for the rest. + """ + images = { + image + for final in finals + for rollout in final.feedback_result["rollouts"] + if (image := rollout["extra"].get("verifier_image")) is not None + } + if not images: + return None + return ", ".join(sorted(images)) + @override async def report(self, finals, fails): total = len(finals) + len(fails) @@ -573,6 +604,12 @@ async def report(self, finals, fails): SCORE_KEY_FIELD: "pass@1", DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, } + verifier_image = self._verifier_image(finals) + if verifier_image is not None: + # **Omitted rather than blanked** when the evaluator reported none: an + # empty string would read as "ran unpinned", which is a different + # claim from "this deployment does not say". + metrics["verifier_image"] = verifier_image if self._n > 1: # At n=1 the rest only restates `pass@1`. metrics.update(rolled) diff --git a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py index 612593a9..df25e48c 100644 --- a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py +++ b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py @@ -55,6 +55,9 @@ _CASES = [{"input": "2\n", "output": "4"}, {"input": "5\n", "output": "10"}] +# What the evaluator reports back as the verifier that scored a rollout. +_IMAGE = "ghcr.io/nuprl/agnostics@sha256:" + "c1" * 32 + def _encode(cases: list[dict]) -> str: """LiveCodeBench's private-test encoding: JSON -> pickle -> zlib -> base64.""" @@ -92,7 +95,11 @@ def json() -> dict: return { "status": True, "msg": "success", - "data": {"n_cases": 1, "n_passed": 1}, + "data": { + "n_cases": 1, + "n_passed": 1, + "verifier_image": _IMAGE, + }, } @@ -389,9 +396,14 @@ def test_k_may_not_exceed_n(): # --------------------------------------------------------------------------- # # report() # --------------------------------------------------------------------------- # -async def _report(msgs: list[str], corrects: list[bool] | None = None): +async def _report( + msgs: list[str], + corrects: list[bool] | None = None, + images: list[str | None] | None = None, +): task = _task() corrects = corrects if corrects is not None else [False] * len(msgs) + images = images if images is not None else [_IMAGE] * len(msgs) try: finals = [ TaskContext( @@ -399,10 +411,16 @@ async def _report(msgs: list[str], corrects: list[bool] | None = None): raw_sample=_raw(), feedback_result=build_judgement_record( None, - [build_rollout_judgement(0, ok, extra={"msg": msg})], + [ + build_rollout_judgement( + 0, ok, extra={"msg": msg, "verifier_image": image} + ) + ], ), ) - for i, (msg, ok) in enumerate(zip(msgs, corrects, strict=True)) + for i, (msg, ok, image) in enumerate( + zip(msgs, corrects, images, strict=True) + ) ] return await task.report(finals, []) finally: @@ -442,6 +460,38 @@ async def test_the_report_declares_its_column_and_the_language_it_measured(): assert report["container_lang"] == "lua" +@pytest.mark.anyio +async def test_the_pinned_verifier_is_recorded_on_the_verdict(): + # A digest-pinned verifier is only as good as the run's record of it: without + # this, nothing on disk says which grader produced the column. + evaluator = await _post_one() + _ = evaluator + + report = await _report(["success"], corrects=[True]) + assert report["verifier_image"] == _IMAGE + + +@pytest.mark.anyio +async def test_two_verifiers_in_one_run_are_both_reported(): + # An evaluator re-pointed mid-run makes the column a mix of two graders. + # Picking one to stand for the rest would hide that. + other = "ghcr.io/nuprl/agnostics@sha256:" + "b4" * 32 + report = await _report( + ["success", "success"], corrects=[True, True], images=[_IMAGE, other] + ) + + assert report["verifier_image"] == ", ".join(sorted([_IMAGE, other])) + + +@pytest.mark.anyio +async def test_an_unreported_verifier_is_omitted_not_blanked(): + # `None` is what a command override yields, where the image is unknowable. + # An empty string would read as "ran unpinned", a different claim. + report = await _report(["success"], corrects=[True], images=[None]) + + assert "verifier_image" not in report + + @pytest.mark.anyio async def test_a_pipeline_failure_counts_against_the_score(): # `DENOMINATOR_REQUESTED`, matching upstream's `pass1`: it divides by rows diff --git a/vendor/code-evaluator/README.md b/vendor/code-evaluator/README.md index 49e856a9..49df9a69 100644 --- a/vendor/code-evaluator/README.md +++ b/vendor/code-evaluator/README.md @@ -331,18 +331,33 @@ all-or-nothing pair (`1/1` or `1/0`); the suite's real size is on the request. The command is **deployment configuration, not a request field** — a caller able to name the image could run any container on this host. It defaults to upstream's -own invocation, with `{lang}` filled from the request: +own invocation, with the image resolved to a **pinned digest**: ```bash -podman run --rm -i --tmpfs /ramdisk:size=512m,exec ghcr.io/nuprl/agnostics:{lang} +podman run --rm -i --tmpfs /ramdisk:size=512m,exec {image} ``` +The verifier decides scores, so it is pinned the way a dataset revision is: a tag +can be moved under a finished leaderboard without anything on disk changing. +`lang` selects a digest from a table covering upstream's eight published tags +(`lua`, `r`, `python`, `jl`, `java`, `cpp`, `ml`, `f90` — note these are **file +extensions**, so Julia is `jl`, OCaml `ml`, Fortran `f90`). A language with no +pinned digest is **refused** with `infra:unpinned-lang` rather than falling back +to a floating tag; add it to `_IMAGE_DIGESTS`, or take responsibility via the +override. All eight are single-platform **linux/amd64** manifests, so the pin +binds the architecture too — on arm64 the command must be overridden. + +The resolved reference comes back as `data.verifier_image` so a caller can record +which verifier produced a verdict. It is `None` under an override unless the +override's template actually contains `{image}`: naming a digest that did not run +is worse than reporting nothing. + Override the whole command with `CODE_EVAL_AGNOSTICS_COMMAND` for a different runtime (upstream also supports `apptainer run --contain --writable-tmpfs .sif`), -a mirrored registry, or a local test double. `lang` must match -`[a-z0-9][a-z0-9_.+-]{0,31}` since it lands in an argv slot. `memory_limit` and -`test.fn_name` are ignored: the container owns its own limits, and the protocol -has no call-based mode. +a mirrored registry, or a local test double; both `{image}` and `{lang}` are +substituted. `lang` must match `[a-z0-9][a-z0-9_.+-]{0,31}` since it lands in an +argv slot. `memory_limit` and `test.fn_name` are ignored: the container owns its +own limits, and the protocol has no call-based mode. ## Resource limits and defaults diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index 27073c58..8c8ab63d 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -401,6 +401,21 @@ Two kinds, and the difference is a decision rather than a status: `CODE_EVAL_AGNOSTICS_COMMAND`, defaulting to upstream's own podman invocation. That is the one place this deviates from upstream's shape rather than its behaviour. + * **The image is pinned by digest, where upstream uses the mutable tag.** The + verifier decides scores, so it gets the treatment a dataset revision gets: + `_IMAGE_DIGESTS` maps each of upstream's eight published tags to a digest + resolved from the registry on 2026-08-23 (each verified against the + manifest's own `Docker-Content-Digest`). A language with no pinned digest is + **refused** (`infra:unpinned-lang`) rather than floated — an unpinned + verifier scores silently, which is the failure the table exists to prevent. + All eight are single-platform **linux/amd64** manifests, so the pin binds the + architecture as well; arm64 needs the override. The resolved reference is + returned as `data.verifier_image` so the verdict's provenance reaches the run + record, and is `None` under an override whose template does not contain + `{image}` — reporting a digest that did not run would be worse than + reporting nothing. Note the tags are **file extensions**, not language names + (`jl`, `ml`, `f90`), which the framework repo's directory names + (`executors/julia`) actively mislead about. * **`infra:` is reported instead of upstream's collapse to `"fail"`.** Upstream turns every harness-side failure (non-zero exit, undecodable stdout) into `result: "fail"` and recovers only the stdin-write case, by matching a @@ -423,6 +438,21 @@ Two kinds, and the difference is a decision rather than a status: Verified against a local stub verifier speaking the protocol (no podman on the dev box): `success` / `fail:wrong-output` / `fail:error` pass through verbatim, - and `infra:timeout` / `infra:bad-lang` / `infra:no-test` all fire. Not yet - upstream -- land in `scitix/code-evaluator` and re-vendor; tests belong there - rather than under `tests/`, which mirrors `sieval/`. + and `infra:timeout` / `infra:bad-lang` / `infra:no-test` all fire. Separately, + on the default (table-driven) path: all four of `lua` / `jl` / `ml` / `f90` + resolve to their digests and report them, `julia` and `rust` are refused as + `infra:unpinned-lang`, and an override templated on `{lang}` reports no image. + Not yet upstream -- land in `scitix/code-evaluator` and re-vendor; tests belong + there rather than under `tests/`, which mirrors `sieval/`. + + **Re-pinning.** The digests are a snapshot. If upstream rebuilds an image, the + table keeps scoring against the old one, which is the intended behaviour -- + moving it is a deliberate act that changes scores. Resolve a new digest with + an anonymous pull token: + + ```bash + TOKEN=$(curl -s "https://ghcr.io/token?scope=repository:nuprl/agnostics:pull&service=ghcr.io" | jq -r .token) + curl -sI -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "https://ghcr.io/v2/nuprl/agnostics/manifests/lua" | grep -i docker-content-digest + ``` diff --git a/vendor/code-evaluator/app/exec_agnostics.py b/vendor/code-evaluator/app/exec_agnostics.py index a41827cd..44ad0147 100644 --- a/vendor/code-evaluator/app/exec_agnostics.py +++ b/vendor/code-evaluator/app/exec_agnostics.py @@ -31,14 +31,41 @@ _COMMAND_ENV_VAR = "CODE_EVAL_AGNOSTICS_COMMAND" -# Upstream's podman invocation, verbatim apart from the `{lang}` placeholder. -# Override the whole command via the env var above for a different runtime -# (upstream also supports `apptainer run --contain --writable-tmpfs .sif`), -# a mirrored registry, or a local test double. -_DEFAULT_COMMAND = ( - "podman run --rm -i --tmpfs /ramdisk:size=512m,exec " - "ghcr.io/nuprl/agnostics:{lang}" -) +_REGISTRY = "ghcr.io/nuprl/agnostics" + +# The verifier is scoring data, so it is pinned the way a dataset revision is: +# by digest, not by the mutable tag upstream's CLI passes. A tag can be moved +# under a finished leaderboard without anything on disk changing. +# +# Resolved from the registry on 2026-08-23 (each digest verified against the +# manifest's own `Docker-Content-Digest`). All eight are single-platform +# **linux/amd64** OCI manifests, not multi-arch indexes, so a digest pin also +# pins the architecture -- on arm64 these will not run and the command has to be +# overridden. +# +# Note the tags are **file extensions**, not language names: Julia is `jl`, +# OCaml `ml`, Fortran `f90`. The framework repo's directories spell the names out +# in full (`executors/julia`), so the tag list is the authority, not the source +# tree. Published: lua, r, python, jl, java, cpp, ml, f90 -- upstream's `c` and +# `rust` executors have no published image. +_IMAGE_DIGESTS = { + "lua": "sha256:c1e437979ceb65a1da46165ab11515d63643ce58fcdb6a31d22fbb3a409117b0", + "r": "sha256:cecf0478d8d090945b296c02515bd46a46107a2fd2860010f94c2a1fdf519c2b", + "python": "sha256:c87079f74d78b9ba514d3431139c6fabb5c109321c95e5fee38a01676d375be3", + "jl": "sha256:b4f00939f8c177818595c1f90b992250c12c159ace7a416cc4410e0c0742a03d", + "java": "sha256:685d73167565b84f5daae1fe2b984f5ea19d60b4a2da50c4a5d7729a74bf5810", + "cpp": "sha256:c154787ba5af5cd873d77e5fedfd4c0f5d79ac3668a9e6e958cd472932e6ceb5", + "ml": "sha256:005b5f7c56ba1f6266b4a7c5e306bdb93cc468f2dd8663480aba6f06187715f0", + "f90": "sha256:ac5487f649ba00c2cdb31451c296e5daf5dac7d3da7422eeaa640f2c98502d13", +} + +# Upstream's podman invocation, with the image resolved to a pinned digest. +# `{image}` is the digest-pinned reference; `{lang}` is still substituted, for an +# override that wants the bare tag. Override the whole command via the env var +# above for a different runtime (upstream also supports +# `apptainer run --contain --writable-tmpfs .sif`), a mirrored registry, or +# a local test double. +_DEFAULT_COMMAND = "podman run --rm -i --tmpfs /ramdisk:size=512m,exec {image}" # `lang` reaches an argv slot, so it is constrained to what a container tag can # be. Not a sandbox -- the command template is trusted and this is not -- but it @@ -51,9 +78,40 @@ _STDIN_WRITE_TIMEOUT = 300.0 -def verifier_command(lang: str) -> list[str]: - template = os.environ.get(_COMMAND_ENV_VAR) or _DEFAULT_COMMAND - return [part.replace("{lang}", lang) for part in shlex.split(template)] +def verifier_command(lang: str) -> tuple[list[str], str | None]: + """Return ``(argv, image)``; ``image`` is the pinned reference that will run. + + ``image`` is ``None`` when the command came from the environment override, + because an arbitrary command's image cannot be known from here -- and a + reported value that might be wrong is worse than an absent one, since this + string travels into the run record as the verdict's provenance. + + Raises ``KeyError`` for a language with no pinned digest, rather than falling + back to the floating tag. An unpinned verifier is the failure this function + exists to prevent, so a new language is added to ``_IMAGE_DIGESTS`` (or run + through the override), never silently floated. + """ + override = os.environ.get(_COMMAND_ENV_VAR) + if override: + digest = _IMAGE_DIGESTS.get(lang) + image = f"{_REGISTRY}@{digest}" if digest else None + argv = [ + part.replace("{image}", image or "").replace("{lang}", lang) + for part in shlex.split(override) + ] + # Report the pinned image only if the override actually asked for it. + # A template using `{lang}` (or naming its own image outright) runs + # something this table cannot vouch for, and naming a digest that did not + # run is worse than reporting nothing -- the value is the verdict's + # provenance in the run record. + return argv, (image if "{image}" in override else None) + + image = f"{_REGISTRY}@{_IMAGE_DIGESTS[lang]}" + argv = [ + part.replace("{image}", image).replace("{lang}", lang) + for part in shlex.split(_DEFAULT_COMMAND) + ] + return argv, image async def execute_agnostics( @@ -62,8 +120,8 @@ async def execute_agnostics( expect_outputs: list[str], lang: str, timeout: float, -) -> tuple[bool, str, ResourceStats]: - """Return ``(passed, msg, stats)``. +) -> tuple[bool, str, ResourceStats, str | None]: + """Return ``(passed, msg, stats, image)``. ``msg`` is the container's ``result`` value verbatim when the container ran and spoke the protocol, and an ``infra:`` code when it did not, so a @@ -73,15 +131,28 @@ async def execute_agnostics( where it is known. It does not change what counts as a pass: only ``"success"`` does, either way. + ``image`` is the digest-pinned reference that ran, for the caller to record + beside the verdict; ``None`` under a command override, where it is unknowable. + ``timeout`` serves as both the container's own ``timeout_s`` and the wall this process is held to -- one number in both roles, as upstream sends it. - A suite whose per-case budgets can sum past the wall is therefore killed - from outside before it can report its own ``fail:timeout``; that tension is - upstream's, and widening the wall here would move scores. + Note the container applies it **per test case** (its harness passes + ``timeout_s`` straight to each ``subprocess.run``), so a multi-case suite can + need many multiples of the wall and is killed from outside first -- which is + why a timing-out submission surfaces as ``infra:timeout`` rather than the + protocol's own ``fail:timeout``. That tension is upstream's, and widening the + wall here would move scores. """ stats = ResourceStats() if not _LANG_PATTERN.match(lang): - return False, f"infra:bad-lang: {lang!r}", stats + return False, f"infra:bad-lang: {lang!r}", stats, None + + try: + command, image = verifier_command(lang) + except KeyError: + # No pinned digest, and no override to take responsibility for one. + # Refusing beats floating a tag: an unpinned verifier scores silently. + return False, f"infra:unpinned-lang: {lang!r}", stats, None payload = json.dumps( { @@ -98,13 +169,13 @@ async def execute_agnostics( try: proc = await asyncio.create_subprocess_exec( - *verifier_command(lang), + *command, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) except Exception as e: - return False, f"infra:spawn: [{type(e).__name__}] {e}", stats + return False, f"infra:spawn: [{type(e).__name__}] {e}", stats, image if proc.pid is not None: stats, stop_event = await monitor_process_resources(proc.pid) @@ -121,7 +192,7 @@ async def execute_agnostics( except (asyncio.TimeoutError, ConnectionError, OSError) as e: proc.kill() await proc.wait() - return False, f"infra:stdin: [{type(e).__name__}] {e}", stats + return False, f"infra:stdin: [{type(e).__name__}] {e}", stats, image try: stdout, stderr = await asyncio.wait_for( @@ -130,23 +201,23 @@ async def execute_agnostics( except asyncio.TimeoutError: proc.kill() await proc.wait() - return False, "infra:timeout", stats + return False, "infra:timeout", stats, image if proc.returncode != 0: detail = stderr.decode(errors="replace").strip()[-500:] - return False, f"infra:exit {proc.returncode}: {detail}", stats + return False, f"infra:exit {proc.returncode}: {detail}", stats, image try: verdict = json.loads(stdout) except json.JSONDecodeError: - return False, "infra:decode", stats + return False, "infra:decode", stats, image if not isinstance(verdict, dict): - return False, "infra:decode", stats + return False, "infra:decode", stats, image result = verdict.get("result") if not isinstance(result, str): - return False, "infra:decode", stats - return result == "success", result, stats + return False, "infra:decode", stats, image + return result == "success", result, stats, image finally: stop_event.set() await asyncio.sleep(0.1) # give the monitor time to finish diff --git a/vendor/code-evaluator/app/server.py b/vendor/code-evaluator/app/server.py index 2aa25acb..2ff0b866 100644 --- a/vendor/code-evaluator/app/server.py +++ b/vendor/code-evaluator/app/server.py @@ -95,6 +95,12 @@ class ResourceMetrics(BaseModel): exit_code: int | None = None timed_out: bool | None = None scenarios_digest: str | None = None + # The one field here that is not a cost. `agnostics` delegates the verdict to + # a pinned container, and which digest scored a rollout is provenance the + # caller has to be able to record -- there is nowhere else on this route to + # put it, since the response model is flat by necessity (see above). `None` + # everywhere else, and under a command override, where it is unknowable. + verifier_image: str | None = None # Direct-run executors, by `lang`. The three hand-rolled modules keep their own @@ -503,7 +509,7 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: # stdin/stdout, and the protocol has no call-based mode to route it to. # `memory_limit` likewise -- the container owns its own limits, and they # belong in the command template rather than in a request field. - ok, msg, stats = await execute_agnostics( + ok, msg, stats, verifier_image = await execute_agnostics( code=sample.code, inputs=sample.test.inputs, expect_outputs=sample.test.outputs, @@ -515,6 +521,7 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: f"evaluate sample '{sample.uuid}' from '{sample.source}', " f"language: {sample.lang}, timeout: {timeout}, " f"cases: {len(sample.test.inputs)}, kwargs: {sample.kwargs}, " + f"image: {verifier_image}, " f"status: {ok}, msg: {msg}, " f"avg_cpu: {stats.cpu_percent:.2f}%, " f"peak_cpu: {stats.peak_cpu_percent:.2f}%, " @@ -534,6 +541,7 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: # suite's real size is on the request, not in this count. n_cases=1, n_passed=int(ok), + verifier_image=verifier_image, ), ) else: From e2c996e859cc10307a4263e01f4c1ded481d8cdd Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 15:31:23 +0800 Subject: [PATCH 04/10] fix(vendor): refuse an unpinned language on the override path too `verifier_command` enforced the digest-pin contract only on the default (podman) path. The override branch returns early with `_IMAGE_DIGESTS.get`, so a language with no pinned digest had `""` substituted into the `{image}` slot instead of raising -- the documented `infra:unpinned-lang` refusal never fired there. podman is the one runtime that needs no override, so *every* other deployment -- upstream's own apptainer included -- ran the branch where the guarantee was missing. Nothing scored silently (an empty argv slot makes most runtimes error), but the operator read whatever the runtime said about a missing argument rather than the diagnostic that names the cause, which is exactly the classifier split this module exists to make. A template containing `{image}` is asking this table for a pinned digest, so it is now refused the same way as the default path. A template *without* `{image}` names its own image and is untouched -- that is the override taking responsibility, which is the point of having one. Verified end-to-end through the real `ghcr.io/nuprl/agnostics` lua verifier at its pinned digest, run under udocker: `rust` now returns `infra:unpinned-lang: 'rust'` where it previously surfaced as `infra:exit 64`. All five override/default x pinned/unpinned combinations behave as documented. VENDORED.md also records what that validation run established, since the file still claimed the container hop was only exercised against a stub: the real verifier returns `success` / `fail:wrong-output` / `fail:error` verbatim, a hand-written correct Lua solution scores `success` on a real decoded LiveCodeBench suite while a deliberately wrong variant does not, and all eight pinned digests still matched the registry on 2026-09-11. Plus the two things podman gives for free that a udocker deployment has to rebuild -- a private workdir per invocation (19 of 40 verdicts were wrong without one) and `--quiet`, because `udocker run` banners land on stdout and this module json-decodes the whole stream. Co-Authored-By: Claude Opus 5 (1M context) --- vendor/code-evaluator/VENDORED.md | 37 +++++++++++++++++---- vendor/code-evaluator/app/exec_agnostics.py | 17 ++++++++-- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index 8c8ab63d..3b8b8a69 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -408,6 +408,13 @@ Two kinds, and the difference is a decision rather than a status: manifest's own `Docker-Content-Digest`). A language with no pinned digest is **refused** (`infra:unpinned-lang`) rather than floated — an unpinned verifier scores silently, which is the failure the table exists to prevent. + The refusal covers the **override path too**, where it originally did not: + a template containing `{image}` is asking this table for a digest, so an + unpinned language raises there as well instead of substituting an empty + string into the argv. That distinction matters because podman is the one + runtime that needs no override — every other deployment, upstream's own + `apptainer` included, runs the branch the guarantee used to skip. A template + *without* `{image}` names its own image and is still left alone. All eight are single-platform **linux/amd64** manifests, so the pin binds the architecture as well; arm64 needs the override. The resolved reference is returned as `data.verifier_image` so the verdict's provenance reaches the run @@ -436,15 +443,33 @@ Two kinds, and the difference is a decision rather than a status: existing `monitor_process_resources` watches the pid it spawned. Reported anyway so `data` is never null, but do not read them as the submission's cost. - Verified against a local stub verifier speaking the protocol (no podman on the - dev box): `success` / `fail:wrong-output` / `fail:error` pass through verbatim, - and `infra:timeout` / `infra:bad-lang` / `infra:no-test` all fire. Separately, - on the default (table-driven) path: all four of `lua` / `jl` / `ml` / `f90` - resolve to their digests and report them, `julia` and `rust` are refused as - `infra:unpinned-lang`, and an override templated on `{lang}` reports no image. + Verified against the **real** `ghcr.io/nuprl/agnostics` lua verifier at its + pinned digest, run under `udocker` (2026-09-11; the dev box has no podman, see + the runtime note below): `success` / `fail:wrong-output` / `fail:error` come + back verbatim from the container, `infra:timeout` / `infra:bad-lang` / + `infra:no-test` all fire, and `data.verifier_image` carries the digest that + ran. A hand-written correct Lua solution scored `success` on a real decoded + LiveCodeBench suite while a deliberately wrong variant scored + `fail:wrong-output`, which is what rules out the decode and whitespace paths. + On the digest table: `lua` / `jl` / `ml` / `f90` resolve and report, `julia` + and `rust` are refused as `infra:unpinned-lang` on the default path *and* + under an `{image}` override, and an override templated on `{lang}` reports no + image. All eight pinned digests still matched the registry on 2026-09-11. Not yet upstream -- land in `scitix/code-evaluator` and re-vendor; tests belong there rather than under `tests/`, which mirrors `sieval/`. + **Running the verifier without podman.** `CODE_EVAL_AGNOSTICS_COMMAND` is the + supported hook, but two things podman gives for free have to be rebuilt. A + udocker container is a *persistent directory* and the Agnostics harness writes + the submission to a fixed path in its cwd, so concurrent requests overwrite + each other's code -- measured 19 of 40 wrong verdicts against one shared + container, 0 of 40 once each invocation got its own bind-mounted workdir. And + `udocker run` prints a banner to **stdout**, which this module json-decodes + whole (as upstream does), so it needs `--quiet`. Point the env var at a + wrapper that asserts the `{image}` it was handed equals the digest its + container was built from; otherwise the reported provenance is a label rather + than a fact. + **Re-pinning.** The digests are a snapshot. If upstream rebuilds an image, the table keeps scoring against the old one, which is the intended behaviour -- moving it is a deliberate act that changes scores. Resolve a new digest with diff --git a/vendor/code-evaluator/app/exec_agnostics.py b/vendor/code-evaluator/app/exec_agnostics.py index 44ad0147..3bbe2dd1 100644 --- a/vendor/code-evaluator/app/exec_agnostics.py +++ b/vendor/code-evaluator/app/exec_agnostics.py @@ -89,11 +89,24 @@ def verifier_command(lang: str) -> tuple[list[str], str | None]: Raises ``KeyError`` for a language with no pinned digest, rather than falling back to the floating tag. An unpinned verifier is the failure this function exists to prevent, so a new language is added to ``_IMAGE_DIGESTS`` (or run - through the override), never silently floated. + through an override that names its own image), never silently floated. That + holds on **both** paths: an override whose template contains ``{image}`` is + asking this table for a digest, so it is refused the same way. """ override = os.environ.get(_COMMAND_ENV_VAR) if override: digest = _IMAGE_DIGESTS.get(lang) + wants_image = "{image}" in override + if wants_image and digest is None: + # The template asked this table for a pinned digest and there is + # none. Substituting "" would hand the runtime an empty argv slot, + # so the operator would read whatever it says about a missing + # argument instead of the one diagnostic that names the cause -- + # and `infra:unpinned-lang` is the split this module exists to make. + # A template WITHOUT `{image}` names its own image and is left + # alone: that is the override taking responsibility, which is the + # point of having one. + raise KeyError(lang) image = f"{_REGISTRY}@{digest}" if digest else None argv = [ part.replace("{image}", image or "").replace("{lang}", lang) @@ -104,7 +117,7 @@ def verifier_command(lang: str) -> tuple[list[str], str | None]: # something this table cannot vouch for, and naming a digest that did not # run is worse than reporting nothing -- the value is the verdict's # provenance in the run record. - return argv, (image if "{image}" in override else None) + return argv, (image if wants_image else None) image = f"{_REGISTRY}@{_IMAGE_DIGESTS[lang]}" argv = [ From e4faaecc014ff305fddecb997dec3b2a833bc647 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 15:31:51 +0800 Subject: [PATCH 05/10] docs(tasks): attribute the 8-space instruction indent to ChatAdapter The module docstring and the test's name both said the rendering keeps "the source docstring's 8-space indentation". Upstream's docstring is indented **4** in the class body, and `Signature.instructions` dedents it to zero before the adapter ever sees it; the 8 comes from `ChatAdapter` joining the instruction lines with a newline plus a hard-coded eight spaces. The frozen bytes were always correct -- independently confirmed by rendering upstream's signature under a real `dspy==3.0.0b2` install, which reproduces both the system and the user message byte-for-byte. Only the stated reason was wrong, and it is the kind of wrong that misleads whoever re-captures the prompt on the next DSPy bump: they would go looking at upstream's source indentation, find 4, and conclude the capture had drifted. The hash test cannot catch this, because it pins this file's own literal against itself. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/tasks/ag_livecodebench_x_0shot_gen.py | 9 ++++++--- .../unit/tasks/test_ag_livecodebench_x_0shot_gen.py | 12 +++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py index 744e0ca8..d7300885 100644 --- a/sieval/tasks/ag_livecodebench_x_0shot_gen.py +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -31,9 +31,12 @@ 2025-08-05. Reproducing it by reading the signature is not possible — two of its properties are only observable by running it: -* the instructions are re-inserted into the system message **still carrying the - source docstring's 8-space indentation**, while ``Signature.instructions`` - itself is dedented; +* the instructions are re-indented by **8 spaces** on their way into the system + message: ``ChatAdapter`` joins their lines with a newline plus a hard-coded + eight spaces. That indent is the adapter's own constant, *not* the 4-space + indent the docstring carries in upstream's class body -- + ``Signature.instructions`` dedents that away to zero first, so reading either + the source or the dedented property tells you the wrong number; * the field list is assembled with a trailing ``": "`` per line and then stripped as a block, so the *first* field line keeps its trailing space and the *last* one loses it. diff --git a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py index df25e48c..a20edf2d 100644 --- a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py +++ b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py @@ -160,11 +160,13 @@ def test_prompt_bytes_match_the_captured_dspy_rendering(): assert digest == _PROMPT_DIGEST -def test_instructions_keep_the_source_docstrings_indentation(): - # DSPy re-inserts the signature docstring into the system message with the - # Python source indentation intact, while `Signature.instructions` itself is - # dedented. Only observable by running it, so it is asserted rather than - # trusted. +def test_instructions_are_reindented_by_the_adapters_own_eight_spaces(): + # DSPy dedents the signature docstring (`Signature.instructions` is flush + # left, whatever the class body used) and `ChatAdapter` then joins its lines + # with a newline plus a hard-coded eight spaces. So the 8 below is the + # adapter's constant, NOT upstream's 4-space source indent -- neither the + # source nor the dedented property predicts it, which is why it is asserted + # against the rendering rather than trusted. system, _ = _render_prompt("Lua", "p") assert ( "objective is: \n Solve the following programming problem" From db74c277b2fc195d0c0580bb0939aee1c232b5d7 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 16:01:12 +0800 Subject: [PATCH 06/10] fix(tasks): adapt the report to the interval API that landed on main The rebase puts this task on top of the metrics rework, whose `sampling_report` now returns interval and unit-declaration keys (`*_ci95` as a list, `ci95_units` as a dict). `metrics` was annotated `dict[str, float | str]`, so `metrics.update(rolled)` no longer type-checks -- two `ty` diagnostics, both in this file, and neither visible until the branch sat on current main. Widening the annotation is the whole of the type fix. The rest adopts the convention that rework introduced and that every sibling pass@1 task now follows (`human_eval_0shot_gen` is the structural twin): `score_key="pass@1"` so the headline's interval is also published as `score_ci95` -- the same estimate over the same values, not a second measurement -- and `ungated_intervals` to lift `score` / `pass@1`'s intervals out of the `n > 1` gate, because the metrics they bracket are published at every budget. `grouping=self.problem_groups(finals)` is inert when one sample is one problem and correct if a run configures `Dataset.repeat`. Verified on a live 20-sample run, which is the only thing that exercises it: the runner validates interval/unit pairing at report-write time and raises there, so a green unit suite says nothing about it. The report now carries `score_ci95` / `pass@1_ci95` = [0.0, 16.84], `n_problems` 20 and `ci95_units` naming `n_problems` for both, with no raise. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/tasks/ag_livecodebench_x_0shot_gen.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py index d7300885..5511e158 100644 --- a/sieval/tasks/ag_livecodebench_x_0shot_gen.py +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -132,6 +132,7 @@ SCORE_KEY_FIELD, health_metrics, sampling_report, + ungated_intervals, ) from sieval.datasets import AgLiveCodeBenchXDatasetSample @@ -592,10 +593,16 @@ async def report(self, finals, fails): # `votes=False`: two correct programs are not one answer, so there is # nothing well-defined to take a majority over (RFC #74). rolled = sampling_report( - finals, n=self._n, k=self._k, denominator=total, votes=False + finals, + n=self._n, + k=self._k, + denominator=total, + votes=False, + score_key="pass@1", + grouping=self.problem_groups(finals), ) pass_at_1 = rolled["pass@1"] - metrics: dict[str, float | str] = { + metrics: dict[str, float | str | list[float] | dict[str, str]] = { "score": pass_at_1, "fails": len(fails), "n_run_errors": n_run_errors, @@ -613,6 +620,9 @@ async def report(self, finals, fails): # empty string would read as "ran unpinned", which is a different # claim from "this deployment does not say". metrics["verifier_image"] = verifier_image + # Outside the n>1 gate, because the metrics they bracket are: `pass@1` + # is published at every budget, and so is the headline copied from it. + metrics |= ungated_intervals(rolled, metrics=("score", "pass@1")) if self._n > 1: # At n=1 the rest only restates `pass@1`. metrics.update(rolled) From 4f26e1edb22e2bae1e50eefec8b47db208b10d87 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 16:08:47 +0800 Subject: [PATCH 07/10] feat(tasks): promote Ag-LiveCodeBench-X to stable on a reproduced anchor The PR shipped `experimental` with its own promotion criterion stated: "one paper cell reproduced within its noise". That has now been done, so both the status and the prose explaining it were stale. **Table 1, Qwen 3 8B x Lua: published 11, measured 10.42** (52/499) -- 0.58 pp, or 0.30 combined binomial sigma (sigma_D = 1.96 pp). Run at upstream's README settings for Qwen 3 (temperature 0.2, top_p 0.95, max_tokens 2048, language "Lua /nothink", one completion), against the real `ghcr.io/nuprl/agnostics` lua verifier at the digest this branch pins -- not a stub. Verdict mix: 74.5% fail:error, 11.7% fail:wrong-output, 10.7% success, 3.1% infra:timeout, which matches the paper's own qualitative finding that base models on low-resource languages fail mostly on syntax and input parsing. Three caveats travel with the number, all in the shipped docstring and `reference_impl.notes` rather than only here, and none of them moves it out of noise: - the paper served `qwen3_8b_awq` (AWQ 4-bit) where this used the only OpenRouter provider for the checkpoint, which reports quantization unknown; - 13 of 499 rollouts never generated (12 provider rate-limits, one content filter) and count as failures, as upstream's `pass1` counts a generation that produced no solution -- biasing the result down from 10.70 to 10.42; - the no-JSONAdapter-retry divergence cost exactly one row (`n_unextracted` 1). Also recorded: a hand-written correct Lua solution scores `success` on a real decoded LiveCodeBench suite while a deliberately wrong variant scores `fail:wrong-output`, which is what rules out the decode and whitespace paths as an explanation for the score; and all eight pinned verifier digests still matched the registry on 2026-09-11. The remaining four languages and every other model in the table are unmeasured and the docstring says so. `meta/index.json` regenerated. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/meta/index.json | 4 +- sieval/tasks/ag_livecodebench_x_0shot_gen.py | 48 ++++++++++++++++---- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 8c3f6ff9..5a5296b5 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -1487,9 +1487,9 @@ "reference_impl": { "source": "nuprl/Ag-LiveCodeBench-X", "url": "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py", - "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, and pinned by DIGEST rather than by upstream's mutable tag -- the digest that scored each rollout comes back on the verdict and is surfaced as report()'s verifier_image; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Verifier tags are upstream's own, which are FILE EXTENSIONS, not language names -- ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased language name would name no image. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). No sieval run has been aligned against a published number yet, which is what status='experimental' records." + "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, and pinned by DIGEST rather than by upstream's mutable tag -- the digest that scored each rollout comes back on the verdict and is surfaced as report()'s verifier_image; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Verifier tags are upstream's own, which are FILE EXTENSIONS, not language names -- ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased language name would name no image. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). ALIGNMENT: Table 1's Qwen 3 8B x Lua cell, published 11, measured 10.42 (52/499) against the real ghcr.io/nuprl/agnostics lua verifier at its pinned digest -- 0.58 pp, or 0.30 combined binomial sigma, at upstream's README settings for Qwen 3 (temperature 0.2, top_p 0.95, max_tokens 2048, language 'Lua /nothink'). Caveats that do not move it out of noise: the paper served qwen3_8b_awq (AWQ 4-bit) where this used a provider reporting unknown quantization; 13 of 499 rollouts never generated (provider rate-limits) and count as failures, biasing the number down to 10.42 from 10.70; and divergence (1) cost exactly one row. The other four languages and every other model in the table are unmeasured." }, - "status": "experimental", + "status": "stable", "reference_kind": "procedure" }, { diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py index 5511e158..4f471aeb 100644 --- a/sieval/tasks/ag_livecodebench_x_0shot_gen.py +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -89,12 +89,30 @@ ``2048`` plus ``/nothink`` for Qwen 3. Those are model-layer settings in sieval (``models:`` / ``infer_args``), never this task's. -``status="experimental"``: the port is faithful on the axes above, but no run has -been aligned against a published number yet. The anchor when one is attempted is -the paper's Ag-LiveCodeBench-X table (Qwen-3 4B/8B, DeepSeek-Coder-6.7B-Instruct, -Phi-4-Mini over Lua, Julia, R, OCaml and Fortran); promoting this task to -``stable`` wants one of those cells reproduced within its noise, at the paper's -own decoding settings and language. +``status="stable"``: one cell of the paper's Ag-LiveCodeBench-X table is +reproduced within its noise. **Qwen 3 8B x Lua, Table 1: published 11, measured +10.42** (52/499) -- a gap of 0.58 pp against a combined binomial +``sigma_D`` of 1.96 pp, so 0.30 sigma. Run at upstream's README settings for +Qwen 3 (``temperature 0.2``, ``top_p 0.95``, ``max_tokens 2048``, +``language="Lua /nothink"``, one completion), against the real +``ghcr.io/nuprl/agnostics`` lua verifier at its pinned digest. + +Three things the number carries, none of which move it out of noise: + +* **Serving precision differs from the paper's.** Upstream's own model name is + ``qwen3_8b_awq`` -- AWQ 4-bit, served locally. The reproduction used + OpenRouter's only Qwen3-8B provider, which reports its quantization as + unknown. Same checkpoint, possibly different precision; recorded rather than + controlled. +* **13 of 499 rollouts never got a generation** (12 provider rate-limits, one + content filter) and count as failures in the denominator, which is what + upstream's ``pass1`` does with a generation that produced no solution. They + bias the number *down*: scoring them at the observed rate would give 10.70. +* **The JSONAdapter divergence above cost exactly one row** (``n_unextracted`` + was 1), so the lower bound is 0.2 pp below the unbounded number. + +The rest of the table is unmeasured: Julia, R, OCaml and Fortran, and every +other model, are anchors that exist but have not been run here. AI-Generated Code - Claude Opus 5 (Anthropic) """ @@ -351,7 +369,7 @@ def _decode_private_test_cases(text: str) -> list[dict[str, str]]: n_shot=0, tags=("english", "code-exec", "multi-language"), model_type="chat", - status="experimental", + status="stable", reference_kind="procedure", reference_impl=ReferenceImpl( source="nuprl/Ag-LiveCodeBench-X", @@ -383,9 +401,19 @@ def _decode_private_test_cases(text: str) -> list[dict[str, str]]: "language name would name no image. Upstream defaults: n=1 completion, " "temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout " "supplied per run (README uses 15s, and sends that one number as " - "BOTH the container's timeout_s and the outer process wall). No " - "sieval run has been aligned against a published number yet, which " - "is what status='experimental' records." + "BOTH the container's timeout_s and the outer process wall). " + "ALIGNMENT: Table 1's Qwen 3 8B x Lua cell, published 11, measured " + "10.42 (52/499) against the real ghcr.io/nuprl/agnostics lua " + "verifier at its pinned digest -- 0.58 pp, or 0.30 combined " + "binomial sigma, at upstream's README settings for Qwen 3 " + "(temperature 0.2, top_p 0.95, max_tokens 2048, language " + "'Lua /nothink'). Caveats that do not move it out of noise: the " + "paper served qwen3_8b_awq (AWQ 4-bit) where this used a provider " + "reporting unknown quantization; 13 of 499 rollouts never " + "generated (provider rate-limits) and count as failures, biasing " + "the number down to 10.42 from 10.70; and divergence (1) cost " + "exactly one row. The other four languages and every other model " + "in the table are unmeasured." ), ), ) From 2bc91b5e4ba42bba17aa13bb7d4d276913d76137 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 17:38:50 +0800 Subject: [PATCH 08/10] fix(tasks): correct the Ag-LiveCodeBench-X deadline, report slot and retry note Three review findings on the same file, one behavioural and two declarations the file makes about itself. **The grade deadline was not outside the evaluator's own budgets.** It was sized `timeout + 300`, but the evaluator spends those two in *series*, not one covering the other: `exec_agnostics` writes the payload under its own 300s `_STDIN_WRITE_TIMEOUT` -- upstream's `stdin_write_timeout`, kept because a decoded suite is tens of MB -- and only then arms the container wall, plus spawn and the monitor's teardown. So the client's deadline expired while the server was still entitled to answer, and the `infra:stdin` verdict that budget exists to report could never arrive. The cost is larger than the lost diagnostic: a verdict fails one rollout, whereas the transport error a too-tight deadline raises propagates out of `feedback` and fails the whole sample, which `DENOMINATOR_REQUESTED` charges as wrong. A narrow tail -- it needs a pathologically slow write, and the 499-problem alignment run never hit it -- but it deflates a score rather than erroring, which is the class worth closing. The two budgets are now named constants and the margin is explicit; the sibling `livecodebench_code_generation_0shot_gen` states the same invariant ("provably outside it") over its own whole-suite wall. The test asserts the inequality as well as the literal, so the reason survives a later change to either budget rather than reading as an arbitrary constant. **The report slot was narrower than what `report()` returns.** It declared `dict[str, float | str]` while `ungated_intervals` contributes `score_ci95` and `pass@1_ci95` (`list[float]`) and `ci95_units` (`dict[str, str]`) -- verified by dumping a real report. All 21 sibling tasks calling `ungated_intervals` declare the full union, and this file's own local at the `metrics` assignment already did; the class slot was missed when the report was adapted to the interval API. Nothing catches it: the stage methods carry no return annotations, so the generic slot is unchecked and a wrong declaration is simply wrong. **The JSONAdapter divergence note overstated upstream's retry.** `__call__` re-raises `ContextWindowExceededError` instead of retrying it. The lower-bound claim is unaffected -- that case fails on both sides -- and the note now says so rather than leaving the reader to find the exception. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/tasks/ag_livecodebench_x_0shot_gen.py | 61 ++++++++++++++----- .../test_ag_livecodebench_x_0shot_gen.py | 15 +++-- 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py index 4f471aeb..2242ac68 100644 --- a/sieval/tasks/ag_livecodebench_x_0shot_gen.py +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -56,15 +56,17 @@ Divergences from upstream, each with what it costs: -* **No JSONAdapter retry.** ``ChatAdapter.__call__`` catches *any* exception from - its own parse and re-issues the whole call through ``JSONAdapter`` — a second, - differently shaped prompt with a JSON response format. This port makes one - call. A reply that omits either marker therefore scores 0 here where upstream - gets one more chance, so this number is a **lower bound**, and the size of the - gap is exactly ``n_unextracted``. Read it before comparing against a published - row; upstream's own README warns that truncation at its default - ``max_tokens=5000`` is common enough to mention, and truncation is precisely - what strips the closing marker. +* **No JSONAdapter retry.** ``ChatAdapter.__call__`` catches every exception its + own call raises, ``ContextWindowExceededError`` excepted, and re-issues the + whole call through ``JSONAdapter`` — a second, differently shaped prompt with a + JSON response format. This port makes one call. A reply that omits either + marker therefore scores 0 here where upstream gets one more chance, so this + number is a **lower bound**, and the size of the gap is exactly + ``n_unextracted``. (The one exception upstream re-raises does not narrow the + gap: a context-window overflow fails on both sides.) Read it before comparing + against a published row; upstream's own README warns that truncation at its + default ``max_tokens=5000`` is common enough to mention, and truncation is + precisely what strips the closing marker. * **The container is run by the evaluator, not by this task, and pinned by digest.** Upstream shells out to ``podman run ... ghcr.io/nuprl/agnostics:`` (or ``apptainer`` for a ``.sif``) from the harness process, against the mutable @@ -217,6 +219,19 @@ # this string here -- the task reads the response's `status`. _INFRA_PREFIX = "infra:" +# The evaluator's own budget for writing the payload (`_STDIN_WRITE_TIMEOUT` in +# `exec_agnostics`), which keeps upstream's `stdin_write_timeout=300` because a +# decoded LiveCodeBench suite is tens of MB. It is spent BEFORE the container +# wall is armed, not instead of it, so one request can legitimately occupy the +# evaluator for this plus `timeout` in series. +_EVALUATOR_STDIN_WRITE_BUDGET = 300.0 + +# On top of those two: container spawn, the resource monitor's teardown, and +# network latency. The HTTP deadline has to sit strictly outside everything the +# evaluator may spend, or the request is abandoned while the server is still +# entitled to answer -- see `feedback`. +_HTTP_DEADLINE_MARGIN = 30.0 + # Upstream tags its published verifiers by **file extension**, not by language # name: `ghcr.io/nuprl/agnostics` ships exactly # `lua, r, python, jl, java, cpp, ml, f90`. So lowercasing the language's first @@ -424,9 +439,13 @@ class AgLiveCodeBenchXZeroShotGenTask( ModelOutput, PredictionRecord, JudgementRecord, - # `float | str`: the report names its own score column and the language - # it measured, neither of which is a measurement. - dict[str, float | str], + # `str`: the report names its own score column and the language it + # measured, neither of which is a measurement. `list[float]` and + # `dict[str, str]`: the interval bounds and the `ci95_units` map that + # `ungated_intervals` contributes -- the slot has to admit them, since + # nothing checks it (the stage methods carry no return annotations, so + # a narrower declaration here is simply wrong rather than caught). + dict[str, float | str | list[float] | dict[str, str]], ] ): def __init__( @@ -526,10 +545,20 @@ async def feedback(self, post, ctx): "test": {"inputs": inputs, "outputs": outputs}, "timeout": self._timeout, }, - # Beyond the container wall, for network latency and for the - # test suite itself: a decoded suite runs to tens of MB, and - # writing it is inside this deadline. - timeout=self._timeout + 300.0, + # Strictly outside what the evaluator may spend on one + # request, which is its stdin-write budget and THEN the + # container wall, in series -- not one covering the other. + # Sized at `timeout + 300` the two are exactly equal, so the + # deadline expires while the server is still entitled to + # answer, and the `infra:stdin` verdict it exists to report + # can never arrive. That matters beyond the diagnostic: a + # verdict fails one rollout, whereas the transport error a + # too-tight deadline raises propagates out of `feedback` and + # fails the whole sample, which `DENOMINATOR_REQUESTED` then + # charges as wrong. + timeout=self._timeout + + _EVALUATOR_STDIN_WRITE_BUDGET + + _HTTP_DEADLINE_MARGIN, ) resp.raise_for_status() res = resp.json() diff --git a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py index a20edf2d..e4dd7405 100644 --- a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py +++ b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py @@ -374,10 +374,17 @@ async def test_the_default_timeout_is_upstreams_fifteen_seconds(): (body,) = evaluator.bodies assert body["timeout"] == 15.0 - # The HTTP deadline must sit outside the container wall, or a timing-out - # submission surfaces as a transport error instead of a verdict. The margin - # is large because writing a decoded suite (tens of MB) is inside it. - assert evaluator.deadlines == [15.0 + 300.0] + # The HTTP deadline must sit STRICTLY outside everything the evaluator may + # spend on one request -- its 300s stdin-write budget and then the container + # wall, in series. At exactly `timeout + 300` the two are equal, so the + # request is abandoned while the server is still entitled to answer and the + # `infra:stdin` verdict never arrives; worse, a transport error fails the + # whole sample where a verdict would have failed one rollout. Asserted as + # the inequality rather than only as a literal, so the reason survives a + # later change to either budget. + (deadline,) = evaluator.deadlines + assert deadline > 15.0 + 300.0 + assert deadline == 15.0 + 300.0 + 30.0 @pytest.mark.anyio From e99a05de0a6583a785a6f19afa28eca430d3f610 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 17:39:13 +0800 Subject: [PATCH 09/10] test(vendor): gate the Agnostics digest-pin refusal on both paths `exec_agnostics` carries the guarantee the whole port rests on -- the verifier is scoring data, so it is pinned by digest and an unpinned language is refused rather than floated -- and had no test. That guarantee has already regressed once: `verifier_command` enforced it on the default podman path while the override branch substituted an empty string into the `{image}` slot, so the documented `infra:unpinned-lang` never fired under any non-podman runtime, upstream's own apptainer included. It was caught by a manual validation run, not by CI, which is the gap this closes. 33 tests, none of which spawns a container: `verifier_command` is a pure function of the environment, and the two `execute_agnostics` cases asserted here both return before the spawn. They cover the digest table against upstream's published tag list (spelled out rather than derived from the table, so a misspelled tag cannot agree with itself), digest well-formedness, and all four path/pin combinations -- default and `{image}`-override refusing an unpinned language, an override naming its own image left alone and reporting no image, and the pinned digest reported on both paths. Verified discriminating by reverse-mutation: restoring the pre-fix module (49f07b1e) fails exactly the five override-refusal cases with `DID NOT RAISE `, and passes the other 28. Placed under `tests/` rather than beside the code, which is what VENDORED.md's convention prescribes for an upstream-bound patch, because `[tool.pytest] testpaths` covers `tests/` only -- a test under `vendor/code-evaluator/tests/` is never collected by this suite and gates nothing. The file's docstring says so, and asks a future re-vendor to leave an equivalent gate behind. This is the placement the `quotebench` source already uses for the same reason. Co-Authored-By: Claude Opus 5 (1M context) --- .../code_evaluator/test_exec_agnostics.py | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 tests/unit/vendor/code_evaluator/test_exec_agnostics.py diff --git a/tests/unit/vendor/code_evaluator/test_exec_agnostics.py b/tests/unit/vendor/code_evaluator/test_exec_agnostics.py new file mode 100644 index 00000000..44b2467f --- /dev/null +++ b/tests/unit/vendor/code_evaluator/test_exec_agnostics.py @@ -0,0 +1,163 @@ +"""Tests for the code-evaluator's agnostics source -- the digest-pin contract. + +`exec_agnostics` is an upstream-bound patch (`vendor/code-evaluator/VENDORED.md`), +whose convention puts tests beside the code so they travel with it. These are +here instead, deliberately: `[tool.pytest] testpaths` covers `tests/` only, so a +test under `vendor/code-evaluator/tests/` is never collected by this suite and +gates nothing. The contract below has already regressed once -- `verifier_command` +enforced the pin on the default path but not on the override -- and was caught by +a manual validation run rather than by CI, which is the gap this file closes. A +re-vendor that moves these upstream should leave an equivalent gate behind. + +Only `app.exec_agnostics` is imported, never `app.server` -- the latter needs +fastapi, which is the evaluator service's dependency and not sieval's. Nothing +here spawns a container: `verifier_command` is a pure function of the environment, +and the two `execute_agnostics` cases asserted below both return before the spawn. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import re +import sys +from pathlib import Path + +import pytest + +# vendor/code-evaluator is a service root, not an installed package — put it on +# sys.path so `app` resolves the way it does inside the service. +_EVALUATOR_DIR = str(Path(__file__).resolve().parents[4] / "vendor" / "code-evaluator") +if _EVALUATOR_DIR not in sys.path: + sys.path.insert(0, _EVALUATOR_DIR) + +from app.exec_agnostics import ( # noqa: E402 # type: ignore[unresolved-import] # vendor/code-evaluator added to sys.path at runtime + _COMMAND_ENV_VAR, + _IMAGE_DIGESTS, + _REGISTRY, + execute_agnostics, + verifier_command, +) + +# What `ghcr.io/nuprl/agnostics` actually publishes. Spelled out rather than +# derived from `_IMAGE_DIGESTS`, so a tag deleted from or misspelled in the table +# fails here instead of the table agreeing with itself. Note these are FILE +# EXTENSIONS, not language names -- Julia is `jl`, OCaml `ml`, Fortran `f90`. +_PUBLISHED_TAGS = frozenset({"lua", "r", "python", "jl", "java", "cpp", "ml", "f90"}) + +# Upstream publishes no image for these, so the table must not carry one. +_UNPINNED = ("rust", "c", "julia", "ocaml", "fortran") + + +@pytest.fixture(autouse=True) +def _no_inherited_override(monkeypatch): + """Every test states its own override, or the absence of one.""" + monkeypatch.delenv(_COMMAND_ENV_VAR, raising=False) + + +# --------------------------------------------------------------------------- # +# The digest table +# --------------------------------------------------------------------------- # +def test_the_table_covers_exactly_upstreams_published_tags() -> None: + assert set(_IMAGE_DIGESTS) == _PUBLISHED_TAGS + + +@pytest.mark.parametrize("lang", sorted(_PUBLISHED_TAGS)) +def test_every_pinned_entry_is_a_wellformed_digest(lang) -> None: + # A tag slipping in where a digest belongs is the failure the pin exists to + # prevent, and it reads as a plausible string everywhere else. + assert re.fullmatch(r"sha256:[0-9a-f]{64}", _IMAGE_DIGESTS[lang]) + + +# --------------------------------------------------------------------------- # +# verifier_command -- the default (podman) path +# --------------------------------------------------------------------------- # +def test_the_default_path_runs_the_pinned_digest_and_reports_it() -> None: + argv, image = verifier_command("lua") + + assert image == f"{_REGISTRY}@{_IMAGE_DIGESTS['lua']}" + # By digest, never `registry:lua` -- a tag moved under a finished + # leaderboard would otherwise change a score with nothing on disk changing. + assert argv[-1] == image + assert ":lua" not in argv[-1] + assert argv[:3] == ["podman", "run", "--rm"] + + +@pytest.mark.parametrize("lang", _UNPINNED) +def test_the_default_path_refuses_an_unpinned_language(lang) -> None: + # Refusing beats floating the mutable tag: an unpinned verifier scores + # silently, which is the whole reason the table exists. + with pytest.raises(KeyError): + verifier_command(lang) + + +# --------------------------------------------------------------------------- # +# verifier_command -- the override path +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("lang", _UNPINNED) +def test_an_image_templated_override_refuses_an_unpinned_language( + monkeypatch, lang +) -> None: + # REGRESSION: this branch once substituted "" into the `{image}` slot instead + # of raising, so the documented `infra:unpinned-lang` refusal never fired + # under any non-podman runtime -- upstream's own apptainer included, since + # podman is the one runtime needing no override. A template asking this table + # for a digest is asking for the pin, and gets the pin's refusal. + monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain {image}") + + with pytest.raises(KeyError): + verifier_command(lang) + + +def test_an_image_templated_override_still_reports_the_pinned_digest( + monkeypatch, +) -> None: + monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain {image}") + + argv, image = verifier_command("jl") + + assert image == f"{_REGISTRY}@{_IMAGE_DIGESTS['jl']}" + assert argv == ["apptainer", "run", "--contain", image] + + +@pytest.mark.parametrize("lang", ["lua", *_UNPINNED]) +def test_an_override_naming_its_own_image_is_left_alone(monkeypatch, lang) -> None: + # The override taking responsibility, which is the point of having one: no + # `{image}`, so the table is never consulted and an unpinned language is not + # refused. It reports NO image even for a language the table could name -- + # what ran is the override's own, and naming a digest that did not run is + # worse than naming none. + monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain /opt/{lang}.sif") + + argv, image = verifier_command(lang) + + assert image is None + assert argv == ["apptainer", "run", "--contain", f"/opt/{lang}.sif"] + + +# --------------------------------------------------------------------------- # +# execute_agnostics -- the refusals that never reach a container +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_an_unpinned_language_is_refused_with_its_own_code() -> None: + # The split this module exists to make: the operator reads the cause, not + # whatever the runtime says about a missing argument. + passed, msg, _stats, image = await execute_agnostics( + code="print(1)", inputs=["1"], expect_outputs=["1"], lang="rust", timeout=15.0 + ) + + assert passed is False + assert msg.startswith("infra:unpinned-lang") + assert image is None + + +@pytest.mark.anyio +@pytest.mark.parametrize("lang", ["../etc", "lua; rm -rf /", "LUA", "", "x" * 33]) +async def test_a_lang_that_cannot_be_a_tag_is_refused_before_any_spawn(lang) -> None: + # `lang` reaches an argv slot. Not a sandbox -- the command template is + # trusted and this is not -- but a tag cannot smuggle in a separator. + passed, msg, _stats, image = await execute_agnostics( + code="print(1)", inputs=["1"], expect_outputs=["1"], lang=lang, timeout=15.0 + ) + + assert passed is False + assert msg.startswith("infra:bad-lang") + assert image is None From 210712a4002631edbb9b75bdcca9d5572dff991f Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 17:47:17 +0800 Subject: [PATCH 10/10] docs(tasks): trim the review-fix comments to what they buy A pass over the prose added by the two preceding commits. No behaviour, no assertions, no test names change -- 78 lines out, 45 in. - **The report slot now uses the family's own wording.** 49 of the 50 tasks declaring this union carry one verbatim three-line comment; the version added here was a bespoke six-line one that also explained why the generic slot goes unchecked. That reasoning belongs to whoever is deciding the convention, not to the fiftieth site following it. - **The deadline reasoning is stated once, at the constants.** It was in both the constant block and the call site, in different words -- "in series", "strictly outside" and "entitled to answer" each appeared twice. The call site now carries no comment at all: `self._timeout + _EVALUATOR_STDIN_WRITE_BUDGET + _HTTP_DEADLINE_MARGIN` already reads as the invariant, which is what naming the budgets was for. This is the sibling's shape, where the wall's derivation is commented and the `+ 2` at the call site is not. - **The `ContextWindowExceededError` note is an inline clause** rather than a trailing parenthetical restating the exception it had just named. - **Test comments that restated the test name are gone**, and the two that duplicated `verifier_command`'s own docstring with it. What is kept is what a reader cannot recover from the code: the REGRESSION marker (so the override case is not "simplified" away), why the published tag list is spelled out instead of derived from the table under test, why an override reports no image even for a language the table could name, and that the `lang` pattern is not a sandbox. Checked the rest of the PR rather than assuming: prose density is 44% for the task, 51% for `exec_agnostics` -- inside the band its recent siblings already occupy (`multipl_e/_base.py` 41%, `exec_quotebench.py` 59%), so the port's own comments are not outliers and are left alone. The module-docstring / `reference_impl.notes` overlap is likewise deliberate: shipped metadata has to carry the divergences on its own, and `meta/index.json` embeds the notes. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/tasks/ag_livecodebench_x_0shot_gen.py | 57 +++++++------------ .../test_ag_livecodebench_x_0shot_gen.py | 13 ++--- .../code_evaluator/test_exec_agnostics.py | 53 +++++++---------- 3 files changed, 45 insertions(+), 78 deletions(-) diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py index 2242ac68..9fa6cdfb 100644 --- a/sieval/tasks/ag_livecodebench_x_0shot_gen.py +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -57,16 +57,15 @@ Divergences from upstream, each with what it costs: * **No JSONAdapter retry.** ``ChatAdapter.__call__`` catches every exception its - own call raises, ``ContextWindowExceededError`` excepted, and re-issues the - whole call through ``JSONAdapter`` — a second, differently shaped prompt with a - JSON response format. This port makes one call. A reply that omits either - marker therefore scores 0 here where upstream gets one more chance, so this - number is a **lower bound**, and the size of the gap is exactly - ``n_unextracted``. (The one exception upstream re-raises does not narrow the - gap: a context-window overflow fails on both sides.) Read it before comparing - against a published row; upstream's own README warns that truncation at its - default ``max_tokens=5000`` is common enough to mention, and truncation is - precisely what strips the closing marker. + own call raises — ``ContextWindowExceededError`` excepted, which fails on both + sides either way — and re-issues the whole call through ``JSONAdapter``, a + second, differently shaped prompt with a JSON response format. This port makes + one call. A reply that omits either marker therefore scores 0 here where + upstream gets one more chance, so this number is a **lower bound**, and the + size of the gap is exactly ``n_unextracted``. Read it before comparing against + a published row; upstream's own README warns that truncation at its default + ``max_tokens=5000`` is common enough to mention, and truncation is precisely + what strips the closing marker. * **The container is run by the evaluator, not by this task, and pinned by digest.** Upstream shells out to ``podman run ... ghcr.io/nuprl/agnostics:`` (or ``apptainer`` for a ``.sif``) from the harness process, against the mutable @@ -219,17 +218,15 @@ # this string here -- the task reads the response's `status`. _INFRA_PREFIX = "infra:" -# The evaluator's own budget for writing the payload (`_STDIN_WRITE_TIMEOUT` in -# `exec_agnostics`), which keeps upstream's `stdin_write_timeout=300` because a -# decoded LiveCodeBench suite is tens of MB. It is spent BEFORE the container -# wall is armed, not instead of it, so one request can legitimately occupy the -# evaluator for this plus `timeout` in series. +# What one request may cost the evaluator, in series: it writes the payload +# under its own 300s budget (`_STDIN_WRITE_TIMEOUT` in `exec_agnostics`, kept +# from upstream because a decoded suite is tens of MB) and only THEN arms the +# container wall. So `timeout + 300` is exactly the worst case, not outside it: +# the deadline expires while the server is still entitled to answer, and the +# `infra:stdin` verdict never arrives -- where a verdict fails one rollout, the +# transport error fails the whole sample. The margin covers spawn, the resource +# monitor's teardown and network latency. _EVALUATOR_STDIN_WRITE_BUDGET = 300.0 - -# On top of those two: container spawn, the resource monitor's teardown, and -# network latency. The HTTP deadline has to sit strictly outside everything the -# evaluator may spend, or the request is abandoned while the server is still -# entitled to answer -- see `feedback`. _HTTP_DEADLINE_MARGIN = 30.0 # Upstream tags its published verifiers by **file extension**, not by language @@ -439,12 +436,9 @@ class AgLiveCodeBenchXZeroShotGenTask( ModelOutput, PredictionRecord, JudgementRecord, - # `str`: the report names its own score column and the language it - # measured, neither of which is a measurement. `list[float]` and - # `dict[str, str]`: the interval bounds and the `ci95_units` map that - # `ungated_intervals` contributes -- the slot has to admit them, since - # nothing checks it (the stage methods carry no return annotations, so - # a narrower declaration here is simply wrong rather than caught). + # `float | str`: the report carries `score_key`, which names a column + # rather than measuring one; `list[float]` carries an interval, and + # `dict[str, str]` the `ci95_units` map naming each interval's unit. dict[str, float | str | list[float] | dict[str, str]], ] ): @@ -545,17 +539,6 @@ async def feedback(self, post, ctx): "test": {"inputs": inputs, "outputs": outputs}, "timeout": self._timeout, }, - # Strictly outside what the evaluator may spend on one - # request, which is its stdin-write budget and THEN the - # container wall, in series -- not one covering the other. - # Sized at `timeout + 300` the two are exactly equal, so the - # deadline expires while the server is still entitled to - # answer, and the `infra:stdin` verdict it exists to report - # can never arrive. That matters beyond the diagnostic: a - # verdict fails one rollout, whereas the transport error a - # too-tight deadline raises propagates out of `feedback` and - # fails the whole sample, which `DENOMINATOR_REQUESTED` then - # charges as wrong. timeout=self._timeout + _EVALUATOR_STDIN_WRITE_BUDGET + _HTTP_DEADLINE_MARGIN, diff --git a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py index e4dd7405..2d115e53 100644 --- a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py +++ b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py @@ -374,14 +374,11 @@ async def test_the_default_timeout_is_upstreams_fifteen_seconds(): (body,) = evaluator.bodies assert body["timeout"] == 15.0 - # The HTTP deadline must sit STRICTLY outside everything the evaluator may - # spend on one request -- its 300s stdin-write budget and then the container - # wall, in series. At exactly `timeout + 300` the two are equal, so the - # request is abandoned while the server is still entitled to answer and the - # `infra:stdin` verdict never arrives; worse, a transport error fails the - # whole sample where a verdict would have failed one rollout. Asserted as - # the inequality rather than only as a literal, so the reason survives a - # later change to either budget. + # The deadline must sit STRICTLY outside the evaluator's own two budgets, + # which it spends in series (300s stdin write, then the container wall) -- + # at exactly `timeout + 300` it expires while the server is still entitled + # to answer. Asserted as the inequality as well as the literal, so the + # reason survives a later change to either budget. (deadline,) = evaluator.deadlines assert deadline > 15.0 + 300.0 assert deadline == 15.0 + 300.0 + 30.0 diff --git a/tests/unit/vendor/code_evaluator/test_exec_agnostics.py b/tests/unit/vendor/code_evaluator/test_exec_agnostics.py index 44b2467f..0420236f 100644 --- a/tests/unit/vendor/code_evaluator/test_exec_agnostics.py +++ b/tests/unit/vendor/code_evaluator/test_exec_agnostics.py @@ -2,17 +2,14 @@ `exec_agnostics` is an upstream-bound patch (`vendor/code-evaluator/VENDORED.md`), whose convention puts tests beside the code so they travel with it. These are -here instead, deliberately: `[tool.pytest] testpaths` covers `tests/` only, so a -test under `vendor/code-evaluator/tests/` is never collected by this suite and -gates nothing. The contract below has already regressed once -- `verifier_command` -enforced the pin on the default path but not on the override -- and was caught by -a manual validation run rather than by CI, which is the gap this file closes. A -re-vendor that moves these upstream should leave an equivalent gate behind. - -Only `app.exec_agnostics` is imported, never `app.server` -- the latter needs -fastapi, which is the evaluator service's dependency and not sieval's. Nothing -here spawns a container: `verifier_command` is a pure function of the environment, -and the two `execute_agnostics` cases asserted below both return before the spawn. +here instead: `[tool.pytest] testpaths` covers `tests/` only, so a test under +`vendor/code-evaluator/tests/` gates nothing. The contract has already regressed +once -- the pin was enforced on the default path but not on the override, caught +by a manual run rather than by CI -- so a re-vendor should leave a gate behind. + +Only `app.exec_agnostics` is imported, never `app.server`, which needs fastapi -- +the evaluator service's dependency, not sieval's. Nothing here spawns a +container: every case below returns before the spawn. AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) """ @@ -37,10 +34,8 @@ verifier_command, ) -# What `ghcr.io/nuprl/agnostics` actually publishes. Spelled out rather than -# derived from `_IMAGE_DIGESTS`, so a tag deleted from or misspelled in the table -# fails here instead of the table agreeing with itself. Note these are FILE -# EXTENSIONS, not language names -- Julia is `jl`, OCaml `ml`, Fortran `f90`. +# What `ghcr.io/nuprl/agnostics` publishes. Spelled out rather than derived from +# `_IMAGE_DIGESTS`, so a misspelled tag fails here instead of agreeing with itself. _PUBLISHED_TAGS = frozenset({"lua", "r", "python", "jl", "java", "cpp", "ml", "f90"}) # Upstream publishes no image for these, so the table must not carry one. @@ -62,8 +57,8 @@ def test_the_table_covers_exactly_upstreams_published_tags() -> None: @pytest.mark.parametrize("lang", sorted(_PUBLISHED_TAGS)) def test_every_pinned_entry_is_a_wellformed_digest(lang) -> None: - # A tag slipping in where a digest belongs is the failure the pin exists to - # prevent, and it reads as a plausible string everywhere else. + # A tag slipping in where a digest belongs reads as a plausible string + # everywhere else, so nothing but this shape check catches it. assert re.fullmatch(r"sha256:[0-9a-f]{64}", _IMAGE_DIGESTS[lang]) @@ -74,17 +69,13 @@ def test_the_default_path_runs_the_pinned_digest_and_reports_it() -> None: argv, image = verifier_command("lua") assert image == f"{_REGISTRY}@{_IMAGE_DIGESTS['lua']}" - # By digest, never `registry:lua` -- a tag moved under a finished - # leaderboard would otherwise change a score with nothing on disk changing. assert argv[-1] == image - assert ":lua" not in argv[-1] + assert ":lua" not in argv[-1] # by digest, never the mutable tag assert argv[:3] == ["podman", "run", "--rm"] @pytest.mark.parametrize("lang", _UNPINNED) def test_the_default_path_refuses_an_unpinned_language(lang) -> None: - # Refusing beats floating the mutable tag: an unpinned verifier scores - # silently, which is the whole reason the table exists. with pytest.raises(KeyError): verifier_command(lang) @@ -97,10 +88,8 @@ def test_an_image_templated_override_refuses_an_unpinned_language( monkeypatch, lang ) -> None: # REGRESSION: this branch once substituted "" into the `{image}` slot instead - # of raising, so the documented `infra:unpinned-lang` refusal never fired - # under any non-podman runtime -- upstream's own apptainer included, since - # podman is the one runtime needing no override. A template asking this table - # for a digest is asking for the pin, and gets the pin's refusal. + # of raising, so the refusal never fired under any non-podman runtime -- + # i.e. everywhere, podman being the one that needs no override. monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain {image}") with pytest.raises(KeyError): @@ -120,11 +109,10 @@ def test_an_image_templated_override_still_reports_the_pinned_digest( @pytest.mark.parametrize("lang", ["lua", *_UNPINNED]) def test_an_override_naming_its_own_image_is_left_alone(monkeypatch, lang) -> None: - # The override taking responsibility, which is the point of having one: no - # `{image}`, so the table is never consulted and an unpinned language is not - # refused. It reports NO image even for a language the table could name -- - # what ran is the override's own, and naming a digest that did not run is - # worse than naming none. + # No `{image}`, so the table is never consulted and an unpinned language is + # not refused -- the override taking responsibility. Note it reports NO image + # even for `lua`, which the table could name: naming a digest that did not + # run is worse than naming none. monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain /opt/{lang}.sif") argv, image = verifier_command(lang) @@ -138,8 +126,7 @@ def test_an_override_naming_its_own_image_is_left_alone(monkeypatch, lang) -> No # --------------------------------------------------------------------------- # @pytest.mark.anyio async def test_an_unpinned_language_is_refused_with_its_own_code() -> None: - # The split this module exists to make: the operator reads the cause, not - # whatever the runtime says about a missing argument. + # The operator reads the cause, not whatever the runtime says downstream. passed, msg, _stats, image = await execute_agnostics( code="print(1)", inputs=["1"], expect_outputs=["1"], lang="rust", timeout=15.0 )