diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index 79ef2bd0..fa8d0d92 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -9,6 +9,10 @@ from .advanced_if import ( AdvancedIFDataset, AdvancedIFDatasetSample, ) +from .ag_livecodebench_x import ( + AgLiveCodeBenchXDataset, + AgLiveCodeBenchXDatasetSample, +) from .agieval import ( AGIEvalDataset, AGIEvalDatasetSample, @@ -259,6 +263,8 @@ __all__ = [ "ARCEasyDatasetSample", "AdvancedIFDataset", "AdvancedIFDatasetSample", + "AgLiveCodeBenchXDataset", + "AgLiveCodeBenchXDatasetSample", "Apex2025Dataset", "Apex2025DatasetSample", "ApexShortlist2025Dataset", diff --git a/sieval/datasets/ag_livecodebench_x.py b/sieval/datasets/ag_livecodebench_x.py new file mode 100644 index 00000000..4185894b --- /dev/null +++ b/sieval/datasets/ag_livecodebench_x.py @@ -0,0 +1,87 @@ +"""Ag-LiveCodeBench-X — the multi-language LiveCodeBench subset from Agnostics. + +Upstream (NUPRL, `Agnostics `_) took +LiveCodeBench v5's 880 problems, kept the **499 that specify and check their +answer purely through stdin/stdout**, and dropped the 381 that ship Python +starter code. What is left is language-agnostic: the same problem statement and +the same test cases score a solution written in any language, which is why the +set can be pointed at Lua, R, Julia, OCaml or Fortran without re-authoring +anything. + +Three columns and no more — there is no ``starter_code``, no ``metadata`` and no +``public_test_cases`` split here, so every case is a private one and the +call-based (``fn_name``) grading path LiveCodeBench also carries cannot arise. +``private_test_cases`` keeps LiveCodeBench's own encoding (base64 -> zlib -> +pickle -> JSON); decoded, the 499 problems are ~8 GB of test data, which is why +it travels compressed. + +Upstream publishes **no license**: the Hub card carries no license field and the +scripts repo has no ``LICENSE`` file, so ``license`` is ``None`` rather than a +guess. See the task's ``reference_impl.notes`` for what that costs in port +fidelity. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +from typing import TypedDict, override + +from datasets import DatasetDict as HFDatasetDict +from datasets import load_dataset + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +AG_LIVECODEBENCH_X_REVISION = "15bef6bf966893163ea9d22f25b7c58fec06a949" + + +class AgLiveCodeBenchXDatasetSample(TypedDict): + question_id: str + question_content: str + private_test_cases: str + + +@sieval_dataset( + name="ag_livecodebench_x", + display_name="Ag-LiveCodeBench-X", + description=( + "LiveCodeBench v5's 499 stdin/stdout problems, prepared by Agnostics " + "for any language." + ), + source=f"hf:nuprl/Ag-LiveCodeBench-X@{AG_LIVECODEBENCH_X_REVISION}", + categories=( + Category(Level1Category.CODE, "CodeGeneration"), + Category(Level1Category.CODE, "MultiLanguageSupport"), + ), + # No `python` tag, unlike `livecodebench_code_generation`: the target + # language is the task's `language` argument, not a property of the data. + tags=("english", "code-exec"), + # Upstream declares none (no Hub license field, no LICENSE in the scripts + # repo). `None` is the accurate record; do not infer one from LiveCodeBench, + # whose own label ('cc') covers a different artifact. + license=None, +) +class AgLiveCodeBenchXDataset(Dataset[AgLiveCodeBenchXDatasetSample]): + @override + def load(self, name_or_path: str, **kwargs) -> HFDatasetDict: + # One call for both a Hub id and a staged directory: the release is plain + # parquet under `data/`, with no loader script and no config to select, + # so `load_dataset` resolves either form on its own. No `trust_remote_code` + # for the same reason -- unlike `livecodebench_code_generation`, whose Hub + # copy ships a builder script. + dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs)) + + # A window that filters everything away would otherwise be reported as a + # score of 0.0 rather than as a mistake. + if any(len(split) == 0 for split in dataset.values()): + raise ValueError( + "Ag-LiveCodeBench-X dataset is empty after loading " + f"(name_or_path={name_or_path!r}); the release has a single " + "'test' split of 499 problems." + ) + + return dataset diff --git a/sieval/meta/index.json b/sieval/meta/index.json index fbecff08..5a5296b5 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -45,6 +45,31 @@ "license": "CC-BY-NC-4.0", "checksums": {} }, + { + "name": "ag_livecodebench_x", + "display_name": "Ag-LiveCodeBench-X", + "description": "LiveCodeBench v5's 499 stdin/stdout problems, prepared by Agnostics for any language.", + "source": [ + "hf:nuprl/Ag-LiveCodeBench-X@15bef6bf966893163ea9d22f25b7c58fec06a949" + ], + "categories": [ + { + "level1": "Code", + "level2": "CodeGeneration" + }, + { + "level1": "Code", + "level2": "MultiLanguageSupport" + } + ], + "tags": [ + "english", + "code-exec" + ], + "deps_group": null, + "license": null, + "checksums": {} + }, { "name": "agieval", "display_name": "AGIEval", @@ -1445,6 +1470,28 @@ "status": "experimental", "reference_kind": "procedure" }, + { + "name": "ag_livecodebench_x_0shot_gen", + "display_name": "Ag-LiveCodeBench-X (0-shot)", + "description": "Competitive programming in an arbitrary language, graded by the Agnostics verifier.", + "dataset": "ag_livecodebench_x", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "code-exec", + "multi-language" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "nuprl/Ag-LiveCodeBench-X", + "url": "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py", + "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, and pinned by DIGEST rather than by upstream's mutable tag -- the digest that scored each rollout comes back on the verdict and is surfaced as report()'s verifier_image; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Verifier tags are upstream's own, which are FILE EXTENSIONS, not language names -- ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased language name would name no image. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). ALIGNMENT: Table 1's Qwen 3 8B x Lua cell, published 11, measured 10.42 (52/499) against the real ghcr.io/nuprl/agnostics lua verifier at its pinned digest -- 0.58 pp, or 0.30 combined binomial sigma, at upstream's README settings for Qwen 3 (temperature 0.2, top_p 0.95, max_tokens 2048, language 'Lua /nothink'). Caveats that do not move it out of noise: the paper served qwen3_8b_awq (AWQ 4-bit) where this used a provider reporting unknown quantization; 13 of 499 rollouts never generated (provider rate-limits) and count as failures, biasing the number down to 10.42 from 10.70; and divergence (1) cost exactly one row. The other four languages and every other model in the table are unmeasured." + }, + "status": "stable", + "reference_kind": "procedure" + }, { "name": "agieval_0shot_gen", "display_name": "AGIEval (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 12a6a673..79df62e8 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -7,6 +7,9 @@ from .aa_lcr_0shot_gen import ( from .advanced_if_0shot_gen import ( AdvancedIFZeroShotGenTask, ) +from .ag_livecodebench_x_0shot_gen import ( + AgLiveCodeBenchXZeroShotGenTask, +) from .agieval_0shot_gen import ( AGIEvalZeroShotGenTask, ) @@ -247,6 +250,7 @@ __all__ = [ "ARCEasyFewShotClpTask", "ARCEasyFewShotPplTask", "AdvancedIFZeroShotGenTask", + "AgLiveCodeBenchXZeroShotGenTask", "Apex2025ZeroShotGenTask", "ApexShortlist2025ZeroShotGenTask", "BRUMO2025ZeroShotGenTask", diff --git a/sieval/tasks/ag_livecodebench_x_0shot_gen.py b/sieval/tasks/ag_livecodebench_x_0shot_gen.py new file mode 100644 index 00000000..9fa6cdfb --- /dev/null +++ b/sieval/tasks/ag_livecodebench_x_0shot_gen.py @@ -0,0 +1,675 @@ +"""Ag-LiveCodeBench-X — 0-shot competitive programming in an arbitrary language. + +Port of `nuprl/Ag-LiveCodeBench-X `_, +the multi-language LiveCodeBench prepared by Agnostics (Boruch-Gruszecki et al., +ICLR 2026; ``arXiv:2508.04865``). The 499 problems are LiveCodeBench v5's +stdin/stdout subset, so the *same* problem and the *same* test cases score a +solution in any language: which language is asked for is an argument, not a +property of the data. + +Two knobs, deliberately independent, because upstream keeps them independent: + +* ``language`` is a free-form string spliced into the prompt (``"Lua"``, + ``"Julia 1.10"``, ``"Lua /nothink"`` — upstream's own README suggests appending + a version or a thinking-disable directive there). +* ``container_lang`` selects the verifier the evaluator runs. Upstream warns that + nothing checks the two against each other, and its published tags are **file + extensions rather than language names** -- ``ghcr.io/nuprl/agnostics`` ships + exactly ``lua, r, python, jl, java, cpp, ml, f90``, so Julia is ``jl``, OCaml + is ``ml`` and Fortran is ``f90``. The default maps those through + ``_CONTAINER_TAG_BY_LANGUAGE`` and otherwise lowercases ``language``'s first + word; override it for a private registry or a hand-built container. + +Both are recorded in ``report()``, because the task name cannot hold them: one +registered task measures a different quantity per language, and a leaderboard +column that does not say which language it ran is not comparable with anything. + +**The prompt is a frozen DSPy rendering, not a string upstream wrote.** Upstream +declares a ``dspy.Signature`` and wraps it in ``dspy.ChainOfThought``; the bytes +that reach the model are whatever ``dspy==3.0.0b2``'s default ``ChatAdapter`` +renders from that declaration, pinned by upstream's ``exclude-newer`` of +2025-08-05. Reproducing it by reading the signature is not possible — two of its +properties are only observable by running it: + +* the instructions are re-indented by **8 spaces** on their way into the system + message: ``ChatAdapter`` joins their lines with a newline plus a hard-coded + eight spaces. That indent is the adapter's own constant, *not* the 4-space + indent the docstring carries in upstream's class body -- + ``Signature.instructions`` dedents that away to zero first, so reading either + the source or the dedented property tells you the wrong number; +* the field list is assembled with a trailing ``": "`` per line and then + stripped as a block, so the *first* field line keeps its trailing space and the + *last* one loses it. + +So the rendering was captured once from ``dspy==3.0.0b2`` and frozen below. +Freezing is the point rather than a shortcut: a runtime DSPy dependency would +let a patch release silently re-word every prompt in the fleet, which is the +opposite of what a pinned benchmark is for. +``tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py`` pins the rendered +bytes by hash. + +**Reimplemented rather than vendored, because upstream ships no license.** The +Hub card declares none and the scripts repo has no ``LICENSE``, so nothing lands +in ``sieval/community/``: the response parser and the code extractor below are +written from observed behaviour and pinned by tests, in the shape +``sieval/community/`` is reserved for when a license permits a mirror. + +Divergences from upstream, each with what it costs: + +* **No JSONAdapter retry.** ``ChatAdapter.__call__`` catches every exception its + own call raises — ``ContextWindowExceededError`` excepted, which fails on both + sides either way — and re-issues the whole call through ``JSONAdapter``, a + second, differently shaped prompt with a JSON response format. This port makes + one call. A reply that omits either marker therefore scores 0 here where + upstream gets one more chance, so this number is a **lower bound**, and the + size of the gap is exactly ``n_unextracted``. Read it before comparing against + a published row; upstream's own README warns that truncation at its default + ``max_tokens=5000`` is common enough to mention, and truncation is precisely + what strips the closing marker. +* **The container is run by the evaluator, not by this task, and pinned by + digest.** Upstream shells out to ``podman run ... ghcr.io/nuprl/agnostics:`` + (or ``apptainer`` for a ``.sif``) from the harness process, against the mutable + tag. Here the task POSTs to sieval's code-evaluator, which runs that same + container server-side from a digest — the verifier is scoring data, so it is + pinned the way a dataset revision is, and a tag moved under a finished + leaderboard cannot change a score silently. The invocation is the evaluator's + deployment config, not a per-request field, so a client cannot name an + arbitrary image to execute; the digest that ran comes back on each verdict and + is surfaced once as ``report()``'s ``verifier_image``. Same verifier, same + protocol, stronger pin, different process boundary. +* **No smoke call.** Upstream sends ``lm("Say this is a test!")`` before the run; + it measures nothing and is not reproduced. +* **No DSPy cache.** Upstream defaults it off (``--enable-dspy-cache`` is opt-in + and forces concurrency 1), so this matches the default rather than diverging. + +Sampling protocol: upstream's ``--num-completions`` defaults to **1**, and +``pass1`` divides successes by rows written — a generation that produced no +solution is still executed and still counted, so the denominator is +``DENOMINATOR_REQUESTED``. Upstream's decoding defaults are ``temperature=0.6``, +``top_p=0.95``, ``max_tokens=5000``, with the README recommending ``0.2`` / +``2048`` plus ``/nothink`` for Qwen 3. Those are model-layer settings in sieval +(``models:`` / ``infer_args``), never this task's. + +``status="stable"``: one cell of the paper's Ag-LiveCodeBench-X table is +reproduced within its noise. **Qwen 3 8B x Lua, Table 1: published 11, measured +10.42** (52/499) -- a gap of 0.58 pp against a combined binomial +``sigma_D`` of 1.96 pp, so 0.30 sigma. Run at upstream's README settings for +Qwen 3 (``temperature 0.2``, ``top_p 0.95``, ``max_tokens 2048``, +``language="Lua /nothink"``, one completion), against the real +``ghcr.io/nuprl/agnostics`` lua verifier at its pinned digest. + +Three things the number carries, none of which move it out of noise: + +* **Serving precision differs from the paper's.** Upstream's own model name is + ``qwen3_8b_awq`` -- AWQ 4-bit, served locally. The reproduction used + OpenRouter's only Qwen3-8B provider, which reports its quantization as + unknown. Same checkpoint, possibly different precision; recorded rather than + controlled. +* **13 of 499 rollouts never got a generation** (12 provider rate-limits, one + content filter) and count as failures in the denominator, which is what + upstream's ``pass1`` does with a generation that produced no solution. They + bias the number *down*: scoring them at the observed rate would give 10.70. +* **The JSONAdapter divergence above cost exactly one row** (``n_unextracted`` + was 1), so the lower bound is 0.2 pp below the unbounded number. + +The rest of the table is unmeasured: Julia, R, OCaml and Fortran, and every +other model, are anchors that exist but have not been run here. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import base64 +import json +import os +import pickle +import re +import time +import zlib +from typing import override + +import httpx +from loguru import logger + +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + PredictionRecord, + PromptRecord, + ReferenceImpl, + RolloutJudgement, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, + health_metrics, + sampling_report, + ungated_intervals, +) +from sieval.datasets import AgLiveCodeBenchXDatasetSample + +# --- The frozen DSPy 3.0.0b2 ChatAdapter rendering ------------------------- +# +# Captured from `ChatAdapter().format(ChainOfThought(SolveProblem).predict.signature, +# [], inputs)` under upstream's pinned resolution. Written as adjacent literals +# with explicit `\n` so that the two lines which legitimately END IN A SPACE +# survive an editor and a linter -- as a triple-quoted block they would not. +# Do not reflow: every byte here is on the wire. +_SYSTEM_MESSAGE = ( + "Your input fields are:\n" + "1. `programming_language` (str): \n" + "2. `problem_statement` (str):\n" + "Your output fields are:\n" + "1. `reasoning` (str): \n" + "2. `solution` (str):\n" + "All interactions will be structured in the following way, " + "with the appropriate values filled in.\n" + "\n" + "[[ ## programming_language ## ]]\n" + "{programming_language}\n" + "\n" + "[[ ## problem_statement ## ]]\n" + "{problem_statement}\n" + "\n" + "[[ ## reasoning ## ]]\n" + "{reasoning}\n" + "\n" + "[[ ## solution ## ]]\n" + "{solution}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Solve the following programming problem using the programming " + "language that I have specified. Use ONLY the\n" + " programming language given below." +) + +# The user turn is assembled by concatenation rather than `format`: the system +# message above contains literal `{...}` placeholders that are part of the wire +# text, and a problem statement is free to contain braces of its own. +_USER_PREFIX = "[[ ## programming_language ## ]]\n" +_USER_INFIX = "\n\n[[ ## problem_statement ## ]]\n" +_USER_SUFFIX = ( + "\n\nRespond with the corresponding output fields, starting with the field " + "`[[ ## reasoning ## ]]`, then `[[ ## solution ## ]]`, and then ending with " + "the marker for `[[ ## completed ## ]]`." +) + +# `dspy.adapters.chat_adapter.field_header_pattern`, verbatim. +_FIELD_HEADER_PATTERN = re.compile(r"\[\[ ## (\w+) ## \]\]") + +# `ChainOfThought` prepends `reasoning` to the signature's own outputs, and +# ChatAdapter requires *every* output field to be present or it raises. +_OUTPUT_FIELDS = frozenset({"reasoning", "solution"}) + +# The evaluator answers with the Agnostics `result` value verbatim when the +# container ran and spoke the protocol, and with an `infra:`-prefixed code when +# it did not. Upstream's own `pass1` reports the same split as `run_error_rate`, +# reading it off a stderr suffix; the prefix is a server-side category instead, +# because a client-side classifier over free text decays silently. +# Whether a `result` counts as a pass is the evaluator's call, not read back from +# this string here -- the task reads the response's `status`. +_INFRA_PREFIX = "infra:" + +# What one request may cost the evaluator, in series: it writes the payload +# under its own 300s budget (`_STDIN_WRITE_TIMEOUT` in `exec_agnostics`, kept +# from upstream because a decoded suite is tens of MB) and only THEN arms the +# container wall. So `timeout + 300` is exactly the worst case, not outside it: +# the deadline expires while the server is still entitled to answer, and the +# `infra:stdin` verdict never arrives -- where a verdict fails one rollout, the +# transport error fails the whole sample. The margin covers spawn, the resource +# monitor's teardown and network latency. +_EVALUATOR_STDIN_WRITE_BUDGET = 300.0 +_HTTP_DEADLINE_MARGIN = 30.0 + +# Upstream tags its published verifiers by **file extension**, not by language +# name: `ghcr.io/nuprl/agnostics` ships exactly +# `lua, r, python, jl, java, cpp, ml, f90`. So lowercasing the language's first +# word -- the obvious derivation, and the one this task shipped first -- names a +# tag that does not exist for three of the five languages the paper reports, and +# every rollout comes back `infra:exit` (no such image). The table is upstream's +# tag list, keyed by what a `language` string plausibly starts with; anything not +# in it falls through to the lowercased first word, which is right for `lua` / +# `r` / `python` / `java` and is the only sensible guess for a container the +# table has never seen. +_CONTAINER_TAG_BY_LANGUAGE = { + "julia": "jl", + "ocaml": "ml", + "fortran": "f90", + "c++": "cpp", + "cplusplus": "cpp", +} + + +def _derive_container_tag(language: str) -> str: + """Upstream's published verifier tag for a prompt-language string. + + Only a default -- ``container_lang`` overrides it, which is what a private + registry or a hand-built container needs. Reads the first word, since a + ``language`` may carry a version or a directive (``"Julia 1.10"``, + ``"Lua /nothink"``). + """ + first_word = language.split()[0].lower() + return _CONTAINER_TAG_BY_LANGUAGE.get(first_word, first_word) + + +def _render_prompt( + language: str, question_content: str +) -> tuple[dict[str, str], dict[str, str]]: + """The two turns DSPy would have sent, system first. + + A tuple rather than a list so the type stays precise: a ``list[dict[str, str]]`` + is not a ``list[JSONValue]`` (``list`` is invariant), and widening the element + type here would cost every caller the ability to index a message. + """ + return ( + {"role": "system", "content": _SYSTEM_MESSAGE}, + { + "role": "user", + "content": _USER_PREFIX + + language + + _USER_INFIX + + question_content + + _USER_SUFFIX, + }, + ) + + +def _parse_chat_adapter_fields(completion: str) -> dict[str, str] | None: + """Reproduce ``ChatAdapter.parse`` for this signature; ``None`` on its raise. + + Upstream wraps the DSPy call in a bare ``except`` that turns any parse + failure into ``solution = None``, so a raise and a miss are the same outcome + there. Faithfully reproduced quirks: + + * the **first** occurrence of a field wins; later ones are ignored; + * content on the same line as a header is kept as that field's first line; + * unknown headers (``completed`` included) and any prose before the first + header are dropped; + * an **indented** header still matches, but the inline-content slice is taken + with an offset measured on the *stripped* line and applied to the + *unstripped* one -- so ``" [[ ## solution ## ]]"`` yields a field whose + first line is the leftover ``"]]"``. That is upstream's behaviour, bug and + all, and changing it here would silently move scores. + + ``parse_value`` is not reproduced because it cannot matter: both fields are + annotated ``str``, for which it is ``str(value)``. + """ + sections: list[tuple[str | None, list[str]]] = [(None, [])] + for line in completion.splitlines(): + match = _FIELD_HEADER_PATTERN.match(line.strip()) + if match: + header = match.group(1) + remaining_content = line[match.end() :].strip() + sections.append((header, [remaining_content] if remaining_content else [])) + else: + sections[-1][1].append(line) + + fields: dict[str, str] = {} + for name, lines in sections: + if name is not None and name not in fields and name in _OUTPUT_FIELDS: + fields[name] = "\n".join(lines).strip() + + # `ChatAdapter` compares the parsed key set against the signature's and + # raises when they differ -- a reply carrying `solution` but no `reasoning` + # is a parse failure, not a partial success. + if fields.keys() != _OUTPUT_FIELDS: + return None + return fields + + +def _extract_code_from_markdown(markdown: str | None) -> str | None: + """Reproduce upstream's ``extract_code_from_markdown``. + + First fenced block wins. Note the two behaviours that are easy to + "improve" by accident: **no** fence at all means the whole reply is treated + as code, and an **unterminated** fence returns ``None``. The + ``# Example usage:`` truncation is a Python-shaped heuristic that upstream + applies to every language, so it applies here too. + """ + if markdown is None: + return None + code_block_start = markdown.find("```") + if code_block_start == -1: + # Assume that the whole string is code. + return markdown + + code_start = code_block_start + 3 + code_block_end = markdown.find("```", code_start) + if code_block_end == -1: + return None + + code = markdown[code_start:code_block_end].strip() + + if "# Example usage:" in code: + code = code.split("# Example usage:")[0] + + # Drop the fence's language tag, which `strip()` above left on line one. + first_newline = code.find("\n") + if first_newline > 0: + code = code[first_newline + 1 :] + + return code.strip() + + +def _decode_private_test_cases(text: str) -> list[dict[str, str]]: + """base64 -> zlib -> pickle -> JSON, as LiveCodeBench encodes and upstream + decodes. No plain-JSON fallback: unlike ``livecodebench/code_generation_lite``, + every row of this release is encoded, and a fallback would turn a corrupt + column into a silently empty suite.""" + cases: list[dict[str, str]] = json.loads( + pickle.loads(zlib.decompress(base64.b64decode(text.encode("utf-8")))) + ) + return cases + + +@sieval_task( + name="ag_livecodebench_x_0shot_gen", + display_name="Ag-LiveCodeBench-X (0-shot)", + description=( + "Competitive programming in an arbitrary language, graded by the " + "Agnostics verifier." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "code-exec", "multi-language"), + model_type="chat", + status="stable", + reference_kind="procedure", + reference_impl=ReferenceImpl( + source="nuprl/Ag-LiveCodeBench-X", + url=( + "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/" + "b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py" + ), + notes=( + "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of " + "upstream's ChainOfThought(SolveProblem) -- captured, not read off " + "the signature, because the 8-space instruction indent and the " + "trailing-space asymmetry in the field list are only observable at " + "runtime; pinned by hash in the task's tests. Response parsing and " + "code extraction are REIMPLEMENTED, not vendored: upstream ships no " + "license (no Hub license field, no LICENSE file). Divergences: (1) " + "upstream's ChatAdapter retries a failed parse through JSONAdapter " + "with a differently shaped prompt, which this port does not -- so a " + "reply missing either marker scores 0 here and the gap is exactly " + "n_unextracted, making this a lower bound; (2) the Agnostics " + "verifier container is run by sieval's code-evaluator rather than " + "from the harness process, and pinned by DIGEST rather than by " + "upstream's mutable tag -- the digest that scored each rollout comes " + "back on the verdict and is surfaced as report()'s verifier_image; " + "(3) upstream's pre-run 'Say this is a test!' " + "smoke call is not reproduced. Verifier tags are upstream's own, " + "which are FILE EXTENSIONS, not language names -- " + "ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, " + "f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased " + "language name would name no image. Upstream defaults: n=1 completion, " + "temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout " + "supplied per run (README uses 15s, and sends that one number as " + "BOTH the container's timeout_s and the outer process wall). " + "ALIGNMENT: Table 1's Qwen 3 8B x Lua cell, published 11, measured " + "10.42 (52/499) against the real ghcr.io/nuprl/agnostics lua " + "verifier at its pinned digest -- 0.58 pp, or 0.30 combined " + "binomial sigma, at upstream's README settings for Qwen 3 " + "(temperature 0.2, top_p 0.95, max_tokens 2048, language " + "'Lua /nothink'). Caveats that do not move it out of noise: the " + "paper served qwen3_8b_awq (AWQ 4-bit) where this used a provider " + "reporting unknown quantization; 13 of 499 rollouts never " + "generated (provider rate-limits) and count as failures, biasing " + "the number down to 10.42 from 10.70; and divergence (1) cost " + "exactly one row. The other four languages and every other model " + "in the table are unmeasured." + ), + ), +) +class AgLiveCodeBenchXZeroShotGenTask( + Task[ + AgLiveCodeBenchXDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + # `float | str`: the report carries `score_key`, which names a column + # rather than measuring one; `list[float]` carries an interval, and + # `dict[str, str]` the `ci95_units` map naming each interval's unit. + dict[str, float | str | list[float] | dict[str, str]], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + language: str | None = None, + container_lang: str | None = None, + k: int = 1, + n: int = 1, + max_concurrency: int = 4, + timeout: float = 15.0, + ): + """*language* is the string spliced into the prompt and has no default: + this benchmark measures a different thing per language, so picking one + silently would let two runs of the same task name mean different things. + + *container_lang* names the Agnostics verifier the evaluator should run; + when omitted it is derived by :func:`_derive_container_tag`, which knows + that upstream tags by file extension (Julia is ``jl``). *timeout* is + the container budget in seconds, sent as upstream sends it -- one number + serving as both the container's own ``timeout_s`` and the wall the + evaluator holds the process to. + """ + super().__init__(dataset=dataset, model=model, name=name) + if k > n: + raise ValueError( + f"pass@{k} needs at least {k} sample(s) per problem, got n={n}." + ) + if not language or not language.strip(): + raise ValueError( + "Ag-LiveCodeBench-X needs an explicit `language` (the string " + "spliced into the prompt, e.g. 'Lua' or 'Julia 1.10'); it is " + "the measurement, not a default. Set `container_lang` too when " + "the verifier tag is not `language`'s first word lowercased." + ) + self._language = language + self._container_lang = container_lang or _derive_container_tag(language) + self._k = k + self._n = n + self._timeout = timeout + self._code_eval_api = os.getenv( + "SIEVAL_CODE_EVAL_API", "http://localhost:11451/evaluations" + ) + self._http_client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=max_concurrency) + ) + + @override + async def preprocess(self, raw, ctx): + system, user = _render_prompt(self._language, raw["question_content"]) + return build_prompt_record( + [system, user], + # No `reference`: the ground truth is a test suite, not a value. + extra={"question_id": raw["question_id"], "language": self._language}, + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + predictions: list[str | None] = [] + for choice in inf.texts: + fields = _parse_chat_adapter_fields(choice) + code = ( + None + if fields is None + else _extract_code_from_markdown(fields["solution"]) + ) + # Empty normalizes to None so `extracted` reports it as a miss, the + # same rule the LiveCodeBench sibling follows. It is still sent to + # the evaluator as "" below, which returns a real verdict rather + # than skipping the rollout. + predictions.append(code or None) + return build_prediction_record(predictions) + + @override + async def feedback(self, post, ctx): + cases = _decode_private_test_cases(ctx.raw_sample["private_test_cases"]) + inputs = [case["input"] for case in cases] + outputs = [case["output"] for case in cases] + + rollouts: list[RolloutJudgement] = [] + for rollout in post["rollouts"]: + idx = rollout["index"] + try: + resp = await self._http_client.post( + self._code_eval_api, + json={ + "uuid": f"{idx}-{time.perf_counter_ns()}", + "source": "agnostics", + "code": rollout.get("prediction") or "", + "lang": self._container_lang, + "test": {"inputs": inputs, "outputs": outputs}, + "timeout": self._timeout, + }, + timeout=self._timeout + + _EVALUATOR_STDIN_WRITE_BUDGET + + _HTTP_DEADLINE_MARGIN, + ) + resp.raise_for_status() + res = resp.json() + # should raise error if no `status` & `msg` field + correct, msg = res["status"], res["msg"] + data = res["data"] or {} + except Exception as e: + logger.warning( + "Evaluation error for sample {}: [{}] {}", + idx, + type(e).__name__, + e, + ) + raise e + + rollouts.append( + build_rollout_judgement( + idx, + correct, + extra={ + # The Agnostics `result` verbatim ("success", + # "fail:wrong-output", "fail:timeout", ...) or an + # `infra:` code. Stored raw; the taxonomy is the + # evaluator's to name, not this client's to guess. + "msg": msg, + "n_cases": data.get("n_cases"), + "n_passed": data.get("n_passed"), + # Which pinned verifier produced this verdict. Recorded + # per rollout rather than once per run because it is the + # evaluator that resolves it, and a deployment could be + # re-pointed mid-run; `None` means the evaluator ran a + # command override, where the image is unknowable. + "verifier_image": data.get("verifier_image"), + "resources": { + key: value + for key, value in data.items() + if key not in ("n_cases", "n_passed", "verifier_image") + }, + }, + ) + ) + + return True, build_judgement_record( + None, # the reference is the test suite below, not a value + rollouts, + extra={ + "question_id": ctx.raw_sample["question_id"], + "n_test_cases": len(cases), + # Every case is private and every problem is stdio: this release + # carries no public split and no `fn_name`, so there is nothing + # to report per problem beyond the count. + "io_mode": "stdio", + "container_lang": self._container_lang, + }, + ) + + @staticmethod + def _verifier_image(finals) -> str | None: + """The pinned verifier behind this column, or ``None`` if unrecorded. + + A run should see exactly one image. More than one means the evaluator was + re-pointed mid-run, which makes the column a mix of two graders -- so all + of them are reported, sorted and joined, rather than one being picked to + stand for the rest. + """ + images = { + image + for final in finals + for rollout in final.feedback_result["rollouts"] + if (image := rollout["extra"].get("verifier_image")) is not None + } + if not images: + return None + return ", ".join(sorted(images)) + + @override + async def report(self, finals, fails): + total = len(finals) + len(fails) + # Upstream's `run_error_rate` counterpart: rollouts whose verdict came + # from the harness failing to run the container rather than from the + # submitted program. Kept as a count, not a rate, so it reads correctly + # against whatever denominator the consumer picks. + n_run_errors = sum( + 1 + for f in finals + for r in f.feedback_result["rollouts"] + if (r["extra"].get("msg") or "").startswith(_INFRA_PREFIX) + ) + # `votes=False`: two correct programs are not one answer, so there is + # nothing well-defined to take a majority over (RFC #74). + rolled = sampling_report( + finals, + n=self._n, + k=self._k, + denominator=total, + votes=False, + score_key="pass@1", + grouping=self.problem_groups(finals), + ) + pass_at_1 = rolled["pass@1"] + metrics: dict[str, float | str | list[float] | dict[str, str]] = { + "score": pass_at_1, + "fails": len(fails), + "n_run_errors": n_run_errors, + "pass@1": pass_at_1, + # Which language this column measured. Not a metric, but the column + # is meaningless without it and the task name cannot carry it. + "language": self._language, + "container_lang": self._container_lang, + SCORE_KEY_FIELD: "pass@1", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + verifier_image = self._verifier_image(finals) + if verifier_image is not None: + # **Omitted rather than blanked** when the evaluator reported none: an + # empty string would read as "ran unpinned", which is a different + # claim from "this deployment does not say". + metrics["verifier_image"] = verifier_image + # Outside the n>1 gate, because the metrics they bracket are: `pass@1` + # is published at every budget, and so is the headline copied from it. + metrics |= ungated_intervals(rolled, metrics=("score", "pass@1")) + if self._n > 1: + # At n=1 the rest only restates `pass@1`. + metrics.update(rolled) + # Outside the gate: extraction health is a fact about the parser, not + # about the draw, and it is what bounds the JSONAdapter divergence. + return metrics | health_metrics(finals) + + @override + async def shutdown(self): + await self._http_client.aclose() diff --git a/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py new file mode 100644 index 00000000..2d115e53 --- /dev/null +++ b/tests/unit/tasks/test_ag_livecodebench_x_0shot_gen.py @@ -0,0 +1,523 @@ +"""Unit tests for the Ag-LiveCodeBench-X 0-shot chat task. + +Three things are pinned here, all of which a reasonable-looking edit would break +silently: + +* **The prompt bytes.** They are a frozen ``dspy==3.0.0b2`` ``ChatAdapter`` + rendering, so nothing in the repo derives them and nothing but a hash can tell + you they still match. The two expectations below that look like typos -- an + 8-space indent inside the system message, a trailing space on one field line + and not the next -- are the rendering, not this file's mistakes. +* **The response parser**, whose expected values were captured by running + ``ChatAdapter.parse`` under that same pin. The indented-header case encodes an + upstream offset bug; "fixing" it would move scores. +* **The code extractor**, whose two surprising branches (no fence at all means + the whole reply is code; an unterminated fence means nothing is) are what a + cleanup usually gets wrong. + +The verifier half lives in the vendored evaluator, a separate deployable with its +own repo, so only what goes on the wire is asserted here. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import base64 +import hashlib +import json +import pickle +import zlib + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import Request, Response +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + TaskContext, + build_judgement_record, + build_prediction_record, + build_rollout_judgement, +) +from sieval.datasets.ag_livecodebench_x import AgLiveCodeBenchXDataset +from sieval.tasks.ag_livecodebench_x_0shot_gen import ( + AgLiveCodeBenchXZeroShotGenTask, + _extract_code_from_markdown, + _parse_chat_adapter_fields, + _render_prompt, +) +from tests.conftest import HandlerTransport + +# sha256 of `system + NUL + user` for the placeholder inputs below, captured from +# `ChatAdapter().format(ChainOfThought(SolveProblem).predict.signature, [], ...)` +# under upstream's pinned resolution (dspy==3.0.0b2, exclude-newer 2025-08-05). +_PROMPT_DIGEST = "69180174fafdd13d7523628d3d1df3ff9c2ef6e026af5c2529f696744ee3ae22" + +_CASES = [{"input": "2\n", "output": "4"}, {"input": "5\n", "output": "10"}] + +# What the evaluator reports back as the verifier that scored a rollout. +_IMAGE = "ghcr.io/nuprl/agnostics@sha256:" + "c1" * 32 + + +def _encode(cases: list[dict]) -> str: + """LiveCodeBench's private-test encoding: JSON -> pickle -> zlib -> base64.""" + return base64.b64encode(zlib.compress(pickle.dumps(json.dumps(cases)))).decode() + + +def _raw() -> dict: + return { + "question_id": "1873_A", + "question_content": "Read an integer and print twice its value.", + "private_test_cases": _encode(_CASES), + } + + +class _StubChatModel(ChatModel): + def __init__(self): + super().__init__(model="mock-chat", api_key="fake") + + def _build_default_transport(self) -> HandlerTransport: + return HandlerTransport(self._stub_arun, "openai_chat") + + async def _stub_arun(self, req: Request) -> Response: + return Response(texts=("[[ ## solution ## ]]\nx",) * req.sampling.n) + + +class _Response: + """Enough of an httpx response for `feedback` -- the submission passed.""" + + @staticmethod + def raise_for_status() -> None: + return None + + @staticmethod + def json() -> dict: + return { + "status": True, + "msg": "success", + "data": { + "n_cases": 1, + "n_passed": 1, + "verifier_image": _IMAGE, + }, + } + + +class _CapturingEvaluator: + """Stands in for the code-eval service, recording what was asked of it.""" + + def __init__(self): + self.bodies: list[dict] = [] + self.deadlines: list[float] = [] + + async def post(self, url, *, json, timeout): + _ = url + self.bodies.append(json) + self.deadlines.append(timeout) + return _Response() + + async def aclose(self) -> None: + return None + + +def _task(**kwargs) -> AgLiveCodeBenchXZeroShotGenTask: + kwargs.setdefault("language", "Lua") + dataset = AgLiveCodeBenchXDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([_raw()])}) + ) + return AgLiveCodeBenchXZeroShotGenTask(dataset, _StubChatModel(), **kwargs) + + +async def _post_one(prediction: str | None = "print(1)", **kwargs): + """Run `feedback` for one rollout and hand back what the evaluator saw.""" + task = _task(**kwargs) + await task._http_client.aclose() # the real client is never used + evaluator = _CapturingEvaluator() + task._http_client = evaluator + try: + await task.feedback( + build_prediction_record([prediction]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + return evaluator + + +# --------------------------------------------------------------------------- # +# The frozen prompt +# --------------------------------------------------------------------------- # +def test_prompt_bytes_match_the_captured_dspy_rendering(): + system, user = _render_prompt("<>", "<>") + assert system["role"] == "system" + assert user["role"] == "user" + + digest = hashlib.sha256( + (system["content"] + "\x00" + user["content"]).encode() + ).hexdigest() + # A mismatch means the constants were edited, not that DSPy changed: the + # rendering is frozen on purpose and there is no runtime DSPy here. + assert digest == _PROMPT_DIGEST + + +def test_instructions_are_reindented_by_the_adapters_own_eight_spaces(): + # DSPy dedents the signature docstring (`Signature.instructions` is flush + # left, whatever the class body used) and `ChatAdapter` then joins its lines + # with a newline plus a hard-coded eight spaces. So the 8 below is the + # adapter's constant, NOT upstream's 4-space source indent -- neither the + # source nor the dedented property predicts it, which is why it is asserted + # against the rendering rather than trusted. + system, _ = _render_prompt("Lua", "p") + assert ( + "objective is: \n Solve the following programming problem" + in (system["content"]) + ) + + +def test_the_field_list_keeps_its_trailing_space_asymmetry(): + # The list is built with a trailing ": " per line and stripped as a block, so + # the first line keeps its trailing space and the last one loses it. Looks + # like a typo; is the wire format. + system, _ = _render_prompt("Lua", "p") + assert "1. `programming_language` (str): \n" in system["content"] + assert "2. `problem_statement` (str):\n" in system["content"] + assert "1. `reasoning` (str): \n" in system["content"] + + +def test_the_problem_statement_may_contain_braces(): + # The system message carries literal `{programming_language}` placeholders as + # wire text, so the user turn is concatenated, never `format`ted. + _, user = _render_prompt("Lua", "print {x} and {{y}}") + assert "print {x} and {{y}}" in user["content"] + + +# --------------------------------------------------------------------------- # +# ChatAdapter response parsing +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + ("completion", "expected"), + [ + pytest.param( + "[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\nprint(1)\n\n" + "[[ ## completed ## ]]\n", + {"reasoning": "r", "solution": "print(1)"}, + id="canonical", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\nprint(1)", + {"reasoning": "r", "solution": "print(1)"}, + id="no-completed-marker", + ), + pytest.param( + "[[ ## solution ## ]]\nprint(1)\n\n[[ ## completed ## ]]\n", + None, + id="missing-reasoning-is-a-parse-failure", + ), + pytest.param( + "[[ ## reasoning ## ]] r\n[[ ## solution ## ]] print(1)\n", + {"reasoning": "r", "solution": "print(1)"}, + id="content-inline-with-the-header", + ), + pytest.param( + " [[ ## reasoning ## ]]\nr\n [[ ## solution ## ]]\nprint(1)\n", + {"reasoning": "]]\nr", "solution": "]]\nprint(1)"}, + id="indented-header-leaks-the-closing-brackets", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n[[ ## solution ## ]]\nfirst\n" + "[[ ## solution ## ]]\nsecond\n", + {"reasoning": "r", "solution": "first"}, + id="first-occurrence-wins", + ), + pytest.param( + "Sure!\n\n[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\nprint(1)\n", + {"reasoning": "r", "solution": "print(1)"}, + id="prose-before-the-first-header-is-dropped", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n[[ ## thoughts ## ]]\nx\n" + "[[ ## solution ## ]]\nprint(1)\n", + {"reasoning": "r", "solution": "print(1)"}, + id="unknown-headers-are-dropped", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr\n\n[[ ## solution ## ]]\n\n" + "[[ ## completed ## ]]\n", + {"reasoning": "r", "solution": ""}, + id="blank-solution-parses-as-empty-not-missing", + ), + pytest.param( + "[[ ## reasoning ## ]]\nr and then the tokens ran ou", + None, + id="truncated-before-the-solution-marker", + ), + ], +) +def test_parse_reproduces_chat_adapter(completion, expected): + # Every expectation here was produced by running dspy==3.0.0b2's own + # ChatAdapter.parse on the same input. + assert _parse_chat_adapter_fields(completion) == expected + + +# --------------------------------------------------------------------------- # +# Code extraction +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + ("markdown", "expected"), + [ + pytest.param("```lua\nprint(1)\n```", "print(1)", id="tagged-fence"), + pytest.param("```\nprint(1)\n```", "print(1)", id="untagged-fence"), + pytest.param( + "```lua\nfirst\n```\ntext\n```lua\nsecond\n```", + "first", + id="first-block-wins", + ), + pytest.param("print(1)", "print(1)", id="no-fence-is-all-code"), + pytest.param("```lua\nprint(1)", None, id="unterminated-fence-is-nothing"), + pytest.param( + "```python\nsolve()\n# Example usage:\nsolve(1)\n```", + "solve()", + id="example-usage-truncation", + ), + pytest.param("", "", id="empty-in-empty-out"), + pytest.param(None, None, id="none-in-none-out"), + ], +) +def test_extract_code_reproduces_upstream(markdown, expected): + assert _extract_code_from_markdown(markdown) == expected + + +# --------------------------------------------------------------------------- # +# language / container_lang +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("language", [None, "", " "]) +def test_language_has_no_default(language): + # The benchmark measures a different thing per language, so a silent default + # would let two runs of one task name mean different things. + with pytest.raises(ValueError, match="explicit `language`"): + _task(language=language) + + +@pytest.mark.parametrize( + ("language", "expected_tag"), + [ + # Tags upstream publishes under the language's own name. + ("Lua", "lua"), + ("Lua /nothink", "lua"), + ("R", "r"), + ("Python", "python"), + ("Java", "java"), + # ...and the ones tagged by FILE EXTENSION instead. Lowercasing the first + # word -- the obvious derivation -- names no published image for these, + # so every rollout would come back `infra:exit`. Three of the five + # languages the paper reports are in this half. + ("Julia 1.10", "jl"), + ("OCaml", "ml"), + ("Fortran", "f90"), + ("C++", "cpp"), + ], +) +def test_container_tag_matches_upstreams_published_tags(language, expected_tag): + # ghcr.io/nuprl/agnostics ships exactly: lua, r, python, jl, java, cpp, ml, f90. + assert _task(language=language)._container_lang == expected_tag + + +def test_an_unknown_language_falls_through_to_its_first_word(): + # No table entry is not an error: a private registry may publish any tag, and + # the first word is the only sensible guess. + assert _task(language="Zig 0.13")._container_lang == "zig" + + +def test_container_tag_can_be_overridden(): + # What a private registry or a hand-built container needs; upstream warns the + # prompt language and the verifier tag are independent. + task = _task(language="Julia", container_lang="julia-1-10-custom") + assert task._container_lang == "julia-1-10-custom" + + +@pytest.mark.anyio +async def test_the_prompt_language_reaches_the_prompt_not_the_container(): + task = _task(language="Lua /nothink", container_lang="lua") + try: + record = await task.preprocess( + _raw(), TaskContext(sample_id=0, raw_sample=_raw()) + ) + finally: + await task.shutdown() + + user = record["prompt"][1]["content"] + assert "Lua /nothink" in user + assert record["extra"]["language"] == "Lua /nothink" + + +# --------------------------------------------------------------------------- # +# What goes on the wire +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_the_request_carries_the_agnostics_source_and_suite(): + evaluator = await _post_one() + + (body,) = evaluator.bodies + assert body["source"] == "agnostics" + assert body["lang"] == "lua" + assert body["code"] == "print(1)" + # Decoded from the sample's own base64/zlib/pickle column, split into the + # parallel lists the evaluator's request model already speaks. + assert body["test"] == {"inputs": ["2\n", "5\n"], "outputs": ["4", "10"]} + # Every problem in this release is stdio, so no `fn_name` is ever sent. + assert "fn_name" not in body["test"] + + +@pytest.mark.anyio +async def test_the_default_timeout_is_upstreams_fifteen_seconds(): + evaluator = await _post_one() + + (body,) = evaluator.bodies + assert body["timeout"] == 15.0 + # The deadline must sit STRICTLY outside the evaluator's own two budgets, + # which it spends in series (300s stdin write, then the container wall) -- + # at exactly `timeout + 300` it expires while the server is still entitled + # to answer. Asserted as the inequality as well as the literal, so the + # reason survives a later change to either budget. + (deadline,) = evaluator.deadlines + assert deadline > 15.0 + 300.0 + assert deadline == 15.0 + 300.0 + 30.0 + + +@pytest.mark.anyio +async def test_an_unextractable_answer_is_still_graded(): + # `None` on disk means "could not extract"; on the wire it is "", so the + # verifier returns a real verdict rather than the rollout being skipped. + evaluator = await _post_one(prediction=None) + + (body,) = evaluator.bodies + assert body["code"] == "" + + +def test_k_may_not_exceed_n(): + with pytest.raises(ValueError, match="pass@2"): + _task(k=2, n=1) + + +# --------------------------------------------------------------------------- # +# report() +# --------------------------------------------------------------------------- # +async def _report( + msgs: list[str], + corrects: list[bool] | None = None, + images: list[str | None] | None = None, +): + task = _task() + corrects = corrects if corrects is not None else [False] * len(msgs) + images = images if images is not None else [_IMAGE] * len(msgs) + try: + finals = [ + TaskContext( + sample_id=i, + raw_sample=_raw(), + feedback_result=build_judgement_record( + None, + [ + build_rollout_judgement( + 0, ok, extra={"msg": msg, "verifier_image": image} + ) + ], + ), + ) + for i, (msg, ok, image) in enumerate( + zip(msgs, corrects, images, strict=True) + ) + ] + return await task.report(finals, []) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_infra_failures_are_counted_separately_from_wrong_answers(): + # Upstream's `run_error_rate` counterpart. It must not touch pass@1: an + # unrunnable container is still a rollout that did not pass. + report = await _report( + ["success", "fail:wrong-output", "infra:timeout", "infra:exit 125: no image"], + corrects=[True, False, False, False], + ) + + assert report["n_run_errors"] == 2 + assert report["pass@1"] == pytest.approx(25.0) + assert report["score"] == report["pass@1"] + + +@pytest.mark.anyio +async def test_a_null_message_does_not_crash_the_counter(): + # An absent `msg` is `None` on disk after serialization drops it. + report = await _report([None]) # type: ignore[list-item] + + assert report["n_run_errors"] == 0 + + +@pytest.mark.anyio +async def test_the_report_declares_its_column_and_the_language_it_measured(): + report = await _report(["success"], corrects=[True]) + + assert report["score_key"] == "pass@1" + assert report["denominator_policy"] == "requested" + # The task name cannot carry the language, so the report has to. + assert report["language"] == "Lua" + assert report["container_lang"] == "lua" + + +@pytest.mark.anyio +async def test_the_pinned_verifier_is_recorded_on_the_verdict(): + # A digest-pinned verifier is only as good as the run's record of it: without + # this, nothing on disk says which grader produced the column. + evaluator = await _post_one() + _ = evaluator + + report = await _report(["success"], corrects=[True]) + assert report["verifier_image"] == _IMAGE + + +@pytest.mark.anyio +async def test_two_verifiers_in_one_run_are_both_reported(): + # An evaluator re-pointed mid-run makes the column a mix of two graders. + # Picking one to stand for the rest would hide that. + other = "ghcr.io/nuprl/agnostics@sha256:" + "b4" * 32 + report = await _report( + ["success", "success"], corrects=[True, True], images=[_IMAGE, other] + ) + + assert report["verifier_image"] == ", ".join(sorted([_IMAGE, other])) + + +@pytest.mark.anyio +async def test_an_unreported_verifier_is_omitted_not_blanked(): + # `None` is what a command override yields, where the image is unknowable. + # An empty string would read as "ran unpinned", a different claim. + report = await _report(["success"], corrects=[True], images=[None]) + + assert "verifier_image" not in report + + +@pytest.mark.anyio +async def test_a_pipeline_failure_counts_against_the_score(): + # `DENOMINATOR_REQUESTED`, matching upstream's `pass1`: it divides by rows + # written, and a generation that produced no solution is still one of them. + task = _task() + try: + finals = [ + TaskContext( + sample_id=0, + raw_sample=_raw(), + feedback_result=build_judgement_record( + None, [build_rollout_judgement(0, True, extra={"msg": "success"})] + ), + ) + ] + report = await task.report( + finals, [TaskContext(sample_id=1, raw_sample=_raw())] + ) + finally: + await task.shutdown() + + assert report["fails"] == 1 + assert report["pass@1"] == pytest.approx(50.0) diff --git a/tests/unit/vendor/code_evaluator/test_exec_agnostics.py b/tests/unit/vendor/code_evaluator/test_exec_agnostics.py new file mode 100644 index 00000000..0420236f --- /dev/null +++ b/tests/unit/vendor/code_evaluator/test_exec_agnostics.py @@ -0,0 +1,150 @@ +"""Tests for the code-evaluator's agnostics source -- the digest-pin contract. + +`exec_agnostics` is an upstream-bound patch (`vendor/code-evaluator/VENDORED.md`), +whose convention puts tests beside the code so they travel with it. These are +here instead: `[tool.pytest] testpaths` covers `tests/` only, so a test under +`vendor/code-evaluator/tests/` gates nothing. The contract has already regressed +once -- the pin was enforced on the default path but not on the override, caught +by a manual run rather than by CI -- so a re-vendor should leave a gate behind. + +Only `app.exec_agnostics` is imported, never `app.server`, which needs fastapi -- +the evaluator service's dependency, not sieval's. Nothing here spawns a +container: every case below returns before the spawn. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import re +import sys +from pathlib import Path + +import pytest + +# vendor/code-evaluator is a service root, not an installed package — put it on +# sys.path so `app` resolves the way it does inside the service. +_EVALUATOR_DIR = str(Path(__file__).resolve().parents[4] / "vendor" / "code-evaluator") +if _EVALUATOR_DIR not in sys.path: + sys.path.insert(0, _EVALUATOR_DIR) + +from app.exec_agnostics import ( # noqa: E402 # type: ignore[unresolved-import] # vendor/code-evaluator added to sys.path at runtime + _COMMAND_ENV_VAR, + _IMAGE_DIGESTS, + _REGISTRY, + execute_agnostics, + verifier_command, +) + +# What `ghcr.io/nuprl/agnostics` publishes. Spelled out rather than derived from +# `_IMAGE_DIGESTS`, so a misspelled tag fails here instead of agreeing with itself. +_PUBLISHED_TAGS = frozenset({"lua", "r", "python", "jl", "java", "cpp", "ml", "f90"}) + +# Upstream publishes no image for these, so the table must not carry one. +_UNPINNED = ("rust", "c", "julia", "ocaml", "fortran") + + +@pytest.fixture(autouse=True) +def _no_inherited_override(monkeypatch): + """Every test states its own override, or the absence of one.""" + monkeypatch.delenv(_COMMAND_ENV_VAR, raising=False) + + +# --------------------------------------------------------------------------- # +# The digest table +# --------------------------------------------------------------------------- # +def test_the_table_covers_exactly_upstreams_published_tags() -> None: + assert set(_IMAGE_DIGESTS) == _PUBLISHED_TAGS + + +@pytest.mark.parametrize("lang", sorted(_PUBLISHED_TAGS)) +def test_every_pinned_entry_is_a_wellformed_digest(lang) -> None: + # A tag slipping in where a digest belongs reads as a plausible string + # everywhere else, so nothing but this shape check catches it. + assert re.fullmatch(r"sha256:[0-9a-f]{64}", _IMAGE_DIGESTS[lang]) + + +# --------------------------------------------------------------------------- # +# verifier_command -- the default (podman) path +# --------------------------------------------------------------------------- # +def test_the_default_path_runs_the_pinned_digest_and_reports_it() -> None: + argv, image = verifier_command("lua") + + assert image == f"{_REGISTRY}@{_IMAGE_DIGESTS['lua']}" + assert argv[-1] == image + assert ":lua" not in argv[-1] # by digest, never the mutable tag + assert argv[:3] == ["podman", "run", "--rm"] + + +@pytest.mark.parametrize("lang", _UNPINNED) +def test_the_default_path_refuses_an_unpinned_language(lang) -> None: + with pytest.raises(KeyError): + verifier_command(lang) + + +# --------------------------------------------------------------------------- # +# verifier_command -- the override path +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("lang", _UNPINNED) +def test_an_image_templated_override_refuses_an_unpinned_language( + monkeypatch, lang +) -> None: + # REGRESSION: this branch once substituted "" into the `{image}` slot instead + # of raising, so the refusal never fired under any non-podman runtime -- + # i.e. everywhere, podman being the one that needs no override. + monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain {image}") + + with pytest.raises(KeyError): + verifier_command(lang) + + +def test_an_image_templated_override_still_reports_the_pinned_digest( + monkeypatch, +) -> None: + monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain {image}") + + argv, image = verifier_command("jl") + + assert image == f"{_REGISTRY}@{_IMAGE_DIGESTS['jl']}" + assert argv == ["apptainer", "run", "--contain", image] + + +@pytest.mark.parametrize("lang", ["lua", *_UNPINNED]) +def test_an_override_naming_its_own_image_is_left_alone(monkeypatch, lang) -> None: + # No `{image}`, so the table is never consulted and an unpinned language is + # not refused -- the override taking responsibility. Note it reports NO image + # even for `lua`, which the table could name: naming a digest that did not + # run is worse than naming none. + monkeypatch.setenv(_COMMAND_ENV_VAR, "apptainer run --contain /opt/{lang}.sif") + + argv, image = verifier_command(lang) + + assert image is None + assert argv == ["apptainer", "run", "--contain", f"/opt/{lang}.sif"] + + +# --------------------------------------------------------------------------- # +# execute_agnostics -- the refusals that never reach a container +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_an_unpinned_language_is_refused_with_its_own_code() -> None: + # The operator reads the cause, not whatever the runtime says downstream. + passed, msg, _stats, image = await execute_agnostics( + code="print(1)", inputs=["1"], expect_outputs=["1"], lang="rust", timeout=15.0 + ) + + assert passed is False + assert msg.startswith("infra:unpinned-lang") + assert image is None + + +@pytest.mark.anyio +@pytest.mark.parametrize("lang", ["../etc", "lua; rm -rf /", "LUA", "", "x" * 33]) +async def test_a_lang_that_cannot_be_a_tag_is_refused_before_any_spawn(lang) -> None: + # `lang` reaches an argv slot. Not a sandbox -- the command template is + # trusted and this is not -- but a tag cannot smuggle in a separator. + passed, msg, _stats, image = await execute_agnostics( + code="print(1)", inputs=["1"], expect_outputs=["1"], lang=lang, timeout=15.0 + ) + + assert passed is False + assert msg.startswith("infra:bad-lang") + assert image is None diff --git a/vendor/code-evaluator/README.md b/vendor/code-evaluator/README.md index ddb6ba22..49df9a69 100644 --- a/vendor/code-evaluator/README.md +++ b/vendor/code-evaluator/README.md @@ -13,6 +13,8 @@ code either by running it directly or by checking it against test cases. - SciCode (python only) - NL2SH-ALFA / InterCode-ALFA (Bash) — **separate route, separate images** - QuoteBench (bash only; fixture + final-state grading, see below) +- Ag-LiveCodeBench-X and anything else speaking the Agnostics protocol (any + language, via a per-language verifier container) HumanEval and SciCode run the submitted code directly; LiveCodeBench needs a function name and compares inputs against expected outputs. A SciCode program @@ -25,6 +27,12 @@ baseline tree with a reset in between. That tree comes from the image, which is upstream's, so there are five of them — one per prepared filesystem — and one service instance hosts exactly one. See "Shell endpoint" below. +`source: "agnostics"` is odd in a different way: it keeps this route's shape but +nothing runs in this process. The request is handed to a container that owns the +language toolchain, which is what makes Lua / R / Julia / OCaml / Fortran +reachable without installing five toolchains here. See "Agnostics protocol" +below. + ## Setup ### Python environment @@ -139,12 +147,14 @@ Fields: - `uuid` - `source`: `"human-eval"` | `"mbpp"` | `"livecodebench"` | `"scicode"` | - `"quotebench"` + `"quotebench"` | `"agnostics"` - `lang`: `python` | `javascript` | `typescript` | `cpp` | `bash` | `perl` — ask `GET /languages` rather than hard-coding this list; ignored by - `quotebench`, which is Bash-only + `quotebench`, which is Bash-only; for `"agnostics"` it is instead + the **container tag** (`lua`, `julia`, `r`, `ocaml`, `fortran`, …) - `code`: the code, as a string — for `quotebench`, the model's reply -- `test`: LiveCodeBench-specific test description (`fn_name` / `inputs` / `outputs`) +- `test`: LiveCodeBench-specific test description (`fn_name` / `inputs` / `outputs`). + `"agnostics"` requires it and uses only `inputs` / `outputs` - `timeout`: float (optional, seconds; defaults below) — a wall for the **whole suite** - `timeout_per_case`: float (optional, seconds) — budget each test case individually, as official LiveCodeBench does @@ -294,6 +304,61 @@ curl -X POST http://localhost:11451/evaluations \ -d '{"uuid":"demo","source":"human-eval","lang":"python","code":"print(42)","memory_limit":1024}' ``` +## Agnostics protocol + +`source: "agnostics"` forwards the submission to a verifier container over one +JSON line in, one JSON line out: + +```text +in { "code": str, "timeout_s": int, + "test_cases": [ { "input": str, "output": str }, ... ] } +out { "result": "success" } + | { "result": "fail:wrong-output", "expected": str, "got": str, "stderr": str } + | { "result": "fail:error", "exit_code": int, "stdout": str, "stderr": str } + | { "result": "fail:timeout", "stdout": str, "stderr": str } + | { "result": "fail:other", "stdout": str, "stderr": str } +``` + +Only `"success"` is a pass. Other result codes and extra fields are allowed by +the protocol, so anything unrecognized is reported as a failure, not an error. +The response `msg` is the container's `result` verbatim, or an `infra:` +code (`spawn` / `stdin` / `timeout` / `exit ` / `decode` / `bad-lang` / +`no-test`) when the container could not be asked — so a caller can tell "the +program is wrong" from "the harness could not run it". + +The verdict covers the whole suite, so `n_cases` / `n_passed` are the +all-or-nothing pair (`1/1` or `1/0`); the suite's real size is on the request. + +The command is **deployment configuration, not a request field** — a caller able +to name the image could run any container on this host. It defaults to upstream's +own invocation, with the image resolved to a **pinned digest**: + +```bash +podman run --rm -i --tmpfs /ramdisk:size=512m,exec {image} +``` + +The verifier decides scores, so it is pinned the way a dataset revision is: a tag +can be moved under a finished leaderboard without anything on disk changing. +`lang` selects a digest from a table covering upstream's eight published tags +(`lua`, `r`, `python`, `jl`, `java`, `cpp`, `ml`, `f90` — note these are **file +extensions**, so Julia is `jl`, OCaml `ml`, Fortran `f90`). A language with no +pinned digest is **refused** with `infra:unpinned-lang` rather than falling back +to a floating tag; add it to `_IMAGE_DIGESTS`, or take responsibility via the +override. All eight are single-platform **linux/amd64** manifests, so the pin +binds the architecture too — on arm64 the command must be overridden. + +The resolved reference comes back as `data.verifier_image` so a caller can record +which verifier produced a verdict. It is `None` under an override unless the +override's template actually contains `{image}`: naming a digest that did not run +is worse than reporting nothing. + +Override the whole command with `CODE_EVAL_AGNOSTICS_COMMAND` for a different +runtime (upstream also supports `apptainer run --contain --writable-tmpfs .sif`), +a mirrored registry, or a local test double; both `{image}` and `{lang}` are +substituted. `lang` must match `[a-z0-9][a-z0-9_.+-]{0,31}` since it lands in an +argv slot. `memory_limit` and `test.fn_name` are ignored: the container owns its +own limits, and the protocol has no call-based mode. + ## Resource limits and defaults ### Timeout @@ -309,6 +374,12 @@ When a request omits `timeout`, these defaults apply: cache; charging it nothing would let a pathological one hang. - livecodebench: 6s + 2s * number of cases, or `(timeout_per_case + 1) * n + 5` when `timeout_per_case` was sent — upstream's own backstop shape +- agnostics: 15s, the value upstream's README uses. It serves as **both** the + container's own `timeout_s` and the wall the process is held to, as upstream + sends it — so a suite whose per-case budgets sum past the wall is killed from + outside before it can report its own `fail:timeout`. Writing the payload gets + its own 300s budget, since a decoded suite runs to tens of MB and a slow write + is not a slow program. For the table-driven languages (cpp / bash / perl) the wall is enforced against the whole **process group**: the program is spawned in its own session and a @@ -383,6 +454,10 @@ whole protocol against a throwaway git repo with no container. - `tests/`: the shell backend's tests (no container required) - `README.md`: this document +The `agnostics` source needs no image file or requirements of its own — the +container it delegates to supplies the toolchain, and only a container runtime +(`podman`, or `apptainer` for a `.sif`) has to exist on the host. + ## Note Isolation is not hardened. Do not run untrusted or high-risk code. diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index d519380f..3b8b8a69 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -381,3 +381,103 @@ Two kinds, and the difference is a decision rather than a status: same 224/224 inside it as on the host, which is what promoted the sieval tasks to `stable`. `docker build` itself stays unexercised; the reproduction recipe is in `tests/acceptance/quotebench/README.md`. +- `app/exec_agnostics.py` (new), `app/server.py`, `README.md` — **the Agnostics + protocol** as `source="agnostics"`, which is what makes a non-Python language + reachable at all. Nothing executes in this process: the request is forwarded to + a per-language verifier container over one JSON line in, one JSON line out + (`{code, timeout_s, test_cases}` -> `{result: "success" | "fail:*", ...}`), and + only `"success"` is a pass. Upstream of the *protocol* is + nuprl/Ag-LiveCodeBench-X at `b7b273ef`; sieval's client is + `ag_livecodebench_x_0shot_gen`. + + Three deliberate choices, each of which a reviewer will want to push back on: + + * **The container command is deployment config, not a request field.** Upstream + passes `--container-name` on its own CLI, which is safe when the harness and + the caller are the same process. Here they are not, so a client able to name + the image could run an arbitrary container on the evaluator host. `lang` is + all the client sends, constrained to `[a-z0-9][a-z0-9_.+-]{0,31}` because it + lands in an argv slot, and the command comes from + `CODE_EVAL_AGNOSTICS_COMMAND`, defaulting to upstream's own podman + invocation. That is the one place this deviates from upstream's shape rather + than its behaviour. + * **The image is pinned by digest, where upstream uses the mutable tag.** The + verifier decides scores, so it gets the treatment a dataset revision gets: + `_IMAGE_DIGESTS` maps each of upstream's eight published tags to a digest + resolved from the registry on 2026-08-23 (each verified against the + manifest's own `Docker-Content-Digest`). A language with no pinned digest is + **refused** (`infra:unpinned-lang`) rather than floated — an unpinned + verifier scores silently, which is the failure the table exists to prevent. + The refusal covers the **override path too**, where it originally did not: + a template containing `{image}` is asking this table for a digest, so an + unpinned language raises there as well instead of substituting an empty + string into the argv. That distinction matters because podman is the one + runtime that needs no override — every other deployment, upstream's own + `apptainer` included, runs the branch the guarantee used to skip. A template + *without* `{image}` names its own image and is still left alone. + All eight are single-platform **linux/amd64** manifests, so the pin binds the + architecture as well; arm64 needs the override. The resolved reference is + returned as `data.verifier_image` so the verdict's provenance reaches the run + record, and is `None` under an override whose template does not contain + `{image}` — reporting a digest that did not run would be worse than + reporting nothing. Note the tags are **file extensions**, not language names + (`jl`, `ml`, `f90`), which the framework repo's directory names + (`executors/julia`) actively mislead about. + * **`infra:` is reported instead of upstream's collapse to `"fail"`.** + Upstream turns every harness-side failure (non-zero exit, undecodable stdout) + into `result: "fail"` and recovers only the stdin-write case, by matching a + stderr suffix. The split is named here, where it is known, rather than left to + a client-side classifier over free text. **It does not change what counts as a + pass** -- only `"success"` does, either way -- so `pass@1` is unaffected and + only the diagnostic count differs (sieval's `n_run_errors` is therefore + broader than upstream's `run_error_rate` numerator). + * **One number in two roles, kept.** `timeout` is sent as both the container's + `timeout_s` and the wall the process is held to, because upstream does that + and widening the wall would move scores. Consequence, confirmed by running it: + the outer wall is armed first, so the container's own `fail:timeout` is + effectively unreachable and a timing-out submission surfaces as + `infra:timeout`. Writing the payload keeps upstream's separate 300s budget + (`stdin_write_timeout`), since a decoded LiveCodeBench suite is tens of MB. + + Resource stats are the podman *client* process's, not the container's -- the + existing `monitor_process_resources` watches the pid it spawned. Reported anyway + so `data` is never null, but do not read them as the submission's cost. + + Verified against the **real** `ghcr.io/nuprl/agnostics` lua verifier at its + pinned digest, run under `udocker` (2026-09-11; the dev box has no podman, see + the runtime note below): `success` / `fail:wrong-output` / `fail:error` come + back verbatim from the container, `infra:timeout` / `infra:bad-lang` / + `infra:no-test` all fire, and `data.verifier_image` carries the digest that + ran. A hand-written correct Lua solution scored `success` on a real decoded + LiveCodeBench suite while a deliberately wrong variant scored + `fail:wrong-output`, which is what rules out the decode and whitespace paths. + On the digest table: `lua` / `jl` / `ml` / `f90` resolve and report, `julia` + and `rust` are refused as `infra:unpinned-lang` on the default path *and* + under an `{image}` override, and an override templated on `{lang}` reports no + image. All eight pinned digests still matched the registry on 2026-09-11. + Not yet upstream -- land in `scitix/code-evaluator` and re-vendor; tests belong + there rather than under `tests/`, which mirrors `sieval/`. + + **Running the verifier without podman.** `CODE_EVAL_AGNOSTICS_COMMAND` is the + supported hook, but two things podman gives for free have to be rebuilt. A + udocker container is a *persistent directory* and the Agnostics harness writes + the submission to a fixed path in its cwd, so concurrent requests overwrite + each other's code -- measured 19 of 40 wrong verdicts against one shared + container, 0 of 40 once each invocation got its own bind-mounted workdir. And + `udocker run` prints a banner to **stdout**, which this module json-decodes + whole (as upstream does), so it needs `--quiet`. Point the env var at a + wrapper that asserts the `{image}` it was handed equals the digest its + container was built from; otherwise the reported provenance is a label rather + than a fact. + + **Re-pinning.** The digests are a snapshot. If upstream rebuilds an image, the + table keeps scoring against the old one, which is the intended behaviour -- + moving it is a deliberate act that changes scores. Resolve a new digest with + an anonymous pull token: + + ```bash + TOKEN=$(curl -s "https://ghcr.io/token?scope=repository:nuprl/agnostics:pull&service=ghcr.io" | jq -r .token) + curl -sI -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "https://ghcr.io/v2/nuprl/agnostics/manifests/lua" | grep -i docker-content-digest + ``` diff --git a/vendor/code-evaluator/app/exec_agnostics.py b/vendor/code-evaluator/app/exec_agnostics.py new file mode 100644 index 00000000..3bbe2dd1 --- /dev/null +++ b/vendor/code-evaluator/app/exec_agnostics.py @@ -0,0 +1,236 @@ +"""Run a submission through an Agnostics verifier container. + +The Agnostics protocol (nuprl/Ag-LiveCodeBench-X) is one JSON line in, one JSON +line out: + + in { "code": str, "timeout_s": int, + "test_cases": [ { "input": str, "output": str }, ... ] } + out { "result": "success" } + | { "result": "fail:wrong-output", "expected": str, "got": str, ... } + | { "result": "fail:error", "exit_code": int, ... } + | { "result": "fail:timeout", ... } + | { "result": "fail:other", ... } + +Only ``"success"`` counts as a pass; the protocol explicitly allows other result +codes and extra fields, so anything unrecognized is a failure rather than an +error. + +The container command is **deployment configuration**, never a request field: a +caller able to name the image could run an arbitrary container on this host. The +default is upstream's own invocation, with ``{lang}`` filled from the request's +``lang``. +""" + +import asyncio +import json +import os +import re +import shlex + +from .resource_monitor import ResourceStats, monitor_process_resources + +_COMMAND_ENV_VAR = "CODE_EVAL_AGNOSTICS_COMMAND" + +_REGISTRY = "ghcr.io/nuprl/agnostics" + +# The verifier is scoring data, so it is pinned the way a dataset revision is: +# by digest, not by the mutable tag upstream's CLI passes. A tag can be moved +# under a finished leaderboard without anything on disk changing. +# +# Resolved from the registry on 2026-08-23 (each digest verified against the +# manifest's own `Docker-Content-Digest`). All eight are single-platform +# **linux/amd64** OCI manifests, not multi-arch indexes, so a digest pin also +# pins the architecture -- on arm64 these will not run and the command has to be +# overridden. +# +# Note the tags are **file extensions**, not language names: Julia is `jl`, +# OCaml `ml`, Fortran `f90`. The framework repo's directories spell the names out +# in full (`executors/julia`), so the tag list is the authority, not the source +# tree. Published: lua, r, python, jl, java, cpp, ml, f90 -- upstream's `c` and +# `rust` executors have no published image. +_IMAGE_DIGESTS = { + "lua": "sha256:c1e437979ceb65a1da46165ab11515d63643ce58fcdb6a31d22fbb3a409117b0", + "r": "sha256:cecf0478d8d090945b296c02515bd46a46107a2fd2860010f94c2a1fdf519c2b", + "python": "sha256:c87079f74d78b9ba514d3431139c6fabb5c109321c95e5fee38a01676d375be3", + "jl": "sha256:b4f00939f8c177818595c1f90b992250c12c159ace7a416cc4410e0c0742a03d", + "java": "sha256:685d73167565b84f5daae1fe2b984f5ea19d60b4a2da50c4a5d7729a74bf5810", + "cpp": "sha256:c154787ba5af5cd873d77e5fedfd4c0f5d79ac3668a9e6e958cd472932e6ceb5", + "ml": "sha256:005b5f7c56ba1f6266b4a7c5e306bdb93cc468f2dd8663480aba6f06187715f0", + "f90": "sha256:ac5487f649ba00c2cdb31451c296e5daf5dac7d3da7422eeaa640f2c98502d13", +} + +# Upstream's podman invocation, with the image resolved to a pinned digest. +# `{image}` is the digest-pinned reference; `{lang}` is still substituted, for an +# override that wants the bare tag. Override the whole command via the env var +# above for a different runtime (upstream also supports +# `apptainer run --contain --writable-tmpfs .sif`), a mirrored registry, or +# a local test double. +_DEFAULT_COMMAND = "podman run --rm -i --tmpfs /ramdisk:size=512m,exec {image}" + +# `lang` reaches an argv slot, so it is constrained to what a container tag can +# be. Not a sandbox -- the command template is trusted and this is not -- but it +# keeps a tag from smuggling in a separator or a path. +_LANG_PATTERN = re.compile(r"\A[a-z0-9][a-z0-9_.+-]{0,31}\Z") + +# Upstream gives writing the payload its own budget, separate from the wall the +# process runs under, because a decoded LiveCodeBench suite is tens of MB and a +# slow write is not a slow program. Its `stdin_write_timeout=300` is kept. +_STDIN_WRITE_TIMEOUT = 300.0 + + +def verifier_command(lang: str) -> tuple[list[str], str | None]: + """Return ``(argv, image)``; ``image`` is the pinned reference that will run. + + ``image`` is ``None`` when the command came from the environment override, + because an arbitrary command's image cannot be known from here -- and a + reported value that might be wrong is worse than an absent one, since this + string travels into the run record as the verdict's provenance. + + Raises ``KeyError`` for a language with no pinned digest, rather than falling + back to the floating tag. An unpinned verifier is the failure this function + exists to prevent, so a new language is added to ``_IMAGE_DIGESTS`` (or run + through an override that names its own image), never silently floated. That + holds on **both** paths: an override whose template contains ``{image}`` is + asking this table for a digest, so it is refused the same way. + """ + override = os.environ.get(_COMMAND_ENV_VAR) + if override: + digest = _IMAGE_DIGESTS.get(lang) + wants_image = "{image}" in override + if wants_image and digest is None: + # The template asked this table for a pinned digest and there is + # none. Substituting "" would hand the runtime an empty argv slot, + # so the operator would read whatever it says about a missing + # argument instead of the one diagnostic that names the cause -- + # and `infra:unpinned-lang` is the split this module exists to make. + # A template WITHOUT `{image}` names its own image and is left + # alone: that is the override taking responsibility, which is the + # point of having one. + raise KeyError(lang) + image = f"{_REGISTRY}@{digest}" if digest else None + argv = [ + part.replace("{image}", image or "").replace("{lang}", lang) + for part in shlex.split(override) + ] + # Report the pinned image only if the override actually asked for it. + # A template using `{lang}` (or naming its own image outright) runs + # something this table cannot vouch for, and naming a digest that did not + # run is worse than reporting nothing -- the value is the verdict's + # provenance in the run record. + return argv, (image if wants_image else None) + + image = f"{_REGISTRY}@{_IMAGE_DIGESTS[lang]}" + argv = [ + part.replace("{image}", image).replace("{lang}", lang) + for part in shlex.split(_DEFAULT_COMMAND) + ] + return argv, image + + +async def execute_agnostics( + code: str, + inputs: list[str], + expect_outputs: list[str], + lang: str, + timeout: float, +) -> tuple[bool, str, ResourceStats, str | None]: + """Return ``(passed, msg, stats, image)``. + + ``msg`` is the container's ``result`` value verbatim when the container ran + and spoke the protocol, and an ``infra:`` code when it did not, so a + client can separate "the program is wrong" from "the harness could not ask". + Upstream collapses every one of the latter to ``"fail"`` and recovers only + the stdin-write case, from a stderr suffix; the split is named here instead, + where it is known. It does not change what counts as a pass: only + ``"success"`` does, either way. + + ``image`` is the digest-pinned reference that ran, for the caller to record + beside the verdict; ``None`` under a command override, where it is unknowable. + + ``timeout`` serves as both the container's own ``timeout_s`` and the wall + this process is held to -- one number in both roles, as upstream sends it. + Note the container applies it **per test case** (its harness passes + ``timeout_s`` straight to each ``subprocess.run``), so a multi-case suite can + need many multiples of the wall and is killed from outside first -- which is + why a timing-out submission surfaces as ``infra:timeout`` rather than the + protocol's own ``fail:timeout``. That tension is upstream's, and widening the + wall here would move scores. + """ + stats = ResourceStats() + if not _LANG_PATTERN.match(lang): + return False, f"infra:bad-lang: {lang!r}", stats, None + + try: + command, image = verifier_command(lang) + except KeyError: + # No pinned digest, and no override to take responsibility for one. + # Refusing beats floating a tag: an unpinned verifier scores silently. + return False, f"infra:unpinned-lang: {lang!r}", stats, None + + payload = json.dumps( + { + "code": code, + # The protocol types it as an int, and upstream passes its + # `--timeout-seconds` straight through. + "timeout_s": int(timeout), + "test_cases": [ + {"input": stdin, "output": stdout} + for stdin, stdout in zip(inputs, expect_outputs, strict=True) + ], + } + ).encode("utf-8") + + try: + proc = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except Exception as e: + return False, f"infra:spawn: [{type(e).__name__}] {e}", stats, image + + if proc.pid is not None: + stats, stop_event = await monitor_process_resources(proc.pid) + else: + stop_event = asyncio.Event() + stop_event.set() + + try: + assert proc.stdin is not None + try: + proc.stdin.write(payload) + await asyncio.wait_for(proc.stdin.drain(), timeout=_STDIN_WRITE_TIMEOUT) + proc.stdin.close() + except (asyncio.TimeoutError, ConnectionError, OSError) as e: + proc.kill() + await proc.wait() + return False, f"infra:stdin: [{type(e).__name__}] {e}", stats, image + + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(), timeout=timeout + ) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + return False, "infra:timeout", stats, image + + if proc.returncode != 0: + detail = stderr.decode(errors="replace").strip()[-500:] + return False, f"infra:exit {proc.returncode}: {detail}", stats, image + + try: + verdict = json.loads(stdout) + except json.JSONDecodeError: + return False, "infra:decode", stats, image + if not isinstance(verdict, dict): + return False, "infra:decode", stats, image + + result = verdict.get("result") + if not isinstance(result, str): + return False, "infra:decode", stats, image + return result == "success", result, stats, image + finally: + stop_event.set() + await asyncio.sleep(0.1) # give the monitor time to finish diff --git a/vendor/code-evaluator/app/server.py b/vendor/code-evaluator/app/server.py index b94204ef..2ff0b866 100644 --- a/vendor/code-evaluator/app/server.py +++ b/vendor/code-evaluator/app/server.py @@ -9,6 +9,7 @@ from loguru import logger from pydantic import BaseModel +from .exec_agnostics import execute_agnostics from .exec_js import execute_code as exec_js from .exec_lang import LANGUAGES, toolchain_entry, toolchain_present from .exec_lang import execute_code as exec_lang @@ -94,6 +95,12 @@ class ResourceMetrics(BaseModel): exit_code: int | None = None timed_out: bool | None = None scenarios_digest: str | None = None + # The one field here that is not a cost. `agnostics` delegates the verdict to + # a pinned container, and which digest scored a rollout is provenance the + # caller has to be able to record -- there is nowhere else on this route to + # put it, since the response model is flat by necessity (see above). `None` + # everywhere else, and under a command override, where it is unknowable. + verifier_image: str | None = None # Direct-run executors, by `lang`. The three hand-rolled modules keep their own @@ -482,6 +489,61 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: scenarios_digest=scenarios_digest(), ), ) + elif sample.source == "agnostics": + # 'agnostics': the Agnostics protocol (nuprl/Ag-LiveCodeBench-X) -- one + # JSON line to a per-language verifier container, one JSON line back. + # `lang` names the container, not an executor in this process, which is + # what makes Lua / R / Julia / OCaml / Fortran reachable at all. + logger.debug(f"code to exec:\n{sample.code}") + + if sample.test is None: + # Unlike 'livecodebench', there is no direct-run reading: the + # protocol has no shape that carries a submission with no suite. + msg = "infra:no-test: source 'agnostics' requires a test suite" + logger.error(msg) + return BasicResponse(status=False, msg=msg, data=None) + + # Upstream's `--timeout-seconds` is required and its README uses 15. + timeout = sample.timeout if sample.timeout is not None else 15.0 + # `test.fn_name` is ignored on purpose: this set is entirely + # stdin/stdout, and the protocol has no call-based mode to route it to. + # `memory_limit` likewise -- the container owns its own limits, and they + # belong in the command template rather than in a request field. + ok, msg, stats, verifier_image = await execute_agnostics( + code=sample.code, + inputs=sample.test.inputs, + expect_outputs=sample.test.outputs, + lang=sample.lang, + timeout=timeout, + ) + + logger.info( + f"evaluate sample '{sample.uuid}' from '{sample.source}', " + f"language: {sample.lang}, timeout: {timeout}, " + f"cases: {len(sample.test.inputs)}, kwargs: {sample.kwargs}, " + f"image: {verifier_image}, " + f"status: {ok}, msg: {msg}, " + f"avg_cpu: {stats.cpu_percent:.2f}%, " + f"peak_cpu: {stats.peak_cpu_percent:.2f}%, " + f"avg_memory: {stats.memory_mb:.2f}MB, " + f"peak_memory: {stats.peak_memory_mb:.2f}MB" + ) + return BasicResponse( + status=ok, + msg=msg, + data=ResourceMetrics( + avg_cpu_percent=stats.cpu_percent, + peak_cpu_percent=stats.peak_cpu_percent, + avg_memory_mb=stats.memory_mb, + peak_memory_mb=stats.peak_memory_mb, + # The verifier returns one verdict for the whole suite, so this + # is the all-or-nothing pair the direct-run modes report. The + # suite's real size is on the request, not in this count. + n_cases=1, + n_passed=int(ok), + verifier_image=verifier_image, + ), + ) else: logger.error(f"not supported data source: {sample.source}") return BasicResponse(