diff --git a/scripts/check_preflight.py b/scripts/check_preflight.py index 4fb11597..8018e27c 100644 --- a/scripts/check_preflight.py +++ b/scripts/check_preflight.py @@ -2370,6 +2370,73 @@ def check_report_declarations(self) -> list[CheckResult]: ) ] + def _resolve_base_class( + self, name: str, tree: ast.Module, path: Path, violations: list[str] + ) -> tuple[ast.ClassDef, ast.Module, Path] | None: + """The class *name* refers to from *path*, with the module holding it. + + Its own module first, and only then an import. A shared base and the + subclasses that parameterize it commonly live in one ``_base.py`` + (``multipl_e``, ``arc``), where there is no ``ImportFrom`` to resolve and + an import-only lookup finds nothing at all. + """ + local = _class_named(tree, name) + if local is not None: + return local, tree, path + target = _resolve_intra_package(tree, name, path, self.project_root) + if target is None or not target.is_file(): + return None + shared = self._parse_task_module(target, violations) + if shared is None: + return None + parent = _class_named(shared, name) + return None if parent is None else (parent, shared, target) + + def _inherited_references( + self, cls: ast.ClassDef, tree: ast.Module, path: Path, violations: list[str] + ) -> list[ast.expr]: + """Every judgement reference up *cls*'s base chain, followed to the end. + + Termination is by a visited set keyed on (module, class name) rather + than by a depth cap: the graph is finite, and a cap is a second number to + get wrong the next time a family nests one level deeper -- which is the + bug this walk replaces. + + A base that resolves to nothing (``Task`` from ``sieval/core``, a stdlib + ``Generic``) is skipped, not reported: the tier is a union, and a hop + that leaves the tree cannot contribute call sites either way. + """ + found: list[ast.expr] = [] + seen: set[tuple[Path, str]] = {(path, cls.name)} + frontier: list[tuple[ast.ClassDef, ast.Module, Path]] = [(cls, tree, path)] + while frontier: + node, node_tree, node_path = frontier.pop() + for base in node.bases: + # A *generic* base is an `ast.Subscript` -- `DemoBase[TSample]` + # -- and the class it names is its `.value`. Unwrapped rather + # than skipped: a leaf that parameterizes its shared base still + # inherits that base's `feedback`, so skipping the subscript form + # drops the leaf out of this check entirely. It then reports as + # `unresolved`, which is a WARN -- so a mis-declaration on such a + # task would be announced as merely unread rather than raised as + # the FAIL it is. + named = base.value if isinstance(base, ast.Subscript) else base + if not isinstance(named, ast.Name): + continue + resolved = self._resolve_base_class( + named.id, node_tree, node_path, violations + ) + if resolved is None: + continue + parent, parent_tree, parent_path = resolved + key = (parent_path, parent.name) + if key in seen: + continue + seen.add(key) + found += _judgement_references(parent) + frontier.append((parent, parent_tree, parent_path)) + return found + def _judgement_references_for( self, cls: ast.ClassDef, tree: ast.Module, path: Path, violations: list[str] ) -> tuple[list[ast.expr], str]: @@ -2382,15 +2449,22 @@ def _judgement_references_for( 2. a helper the class calls, defined here or imported from a sibling inside ``sieval/tasks/`` -- ``arc/``'s four leaves share ``_base.arc_judgement_record``; - 3. a base class resolved the same way -- ``platinum_bench``'s five leaves - inherit ``feedback`` and define none of their own. A *generic* base - counts: ``math_perturb``'s two leaves name theirs as + 3. a base class, followed to the end of the chain -- ``platinum_bench``'s + five leaves inherit ``feedback`` and define none of their own, and + ``multipl_e``'s four sit a level below that again, on per-protocol + subclasses whose shared ``feedback`` is another level up. A *generic* + base counts: ``math_perturb``'s two leaves name theirs as ``MathPerturbZeroShotGenTask[TSample]``, an ``ast.Subscript`` whose ``.value`` is the class. - One level each, not a transitive walk: that covers every task today, and - a deeper chain reports as unresolved (the caller counts it) rather than - passing quietly. + The helper tier is one hop; the base tier is the whole chain. A helper + calling a helper is a refactor away from being inlined, but a class + hierarchy is a shape a benchmark family picks deliberately, and stopping + one level short of the ``feedback`` reports the leaf as ``unresolved`` + -- a WARN meaning "could not read this one", which is indistinguishable + from what it says when the declaration is also wrong. A guard that goes + quiet on the first family to nest twice is worse than no guard, because + the run still ends green. All three are **unioned**; none short-circuits the others. A task whose two branches build the record two ways is the case that matters, and @@ -2423,27 +2497,7 @@ def _judgement_references_for( if fn is not None: helper += _judgement_references(fn) - inherited: list[ast.expr] = [] - for base in cls.bases: - # A *generic* base is an `ast.Subscript` -- `DemoBase[TSample]` -- - # and the class it names is its `.value`. Unwrapped rather than - # skipped: a leaf that parameterizes its shared base still inherits - # that base's `feedback`, so skipping the subscript form drops the - # leaf out of this check entirely. It then reports as `unresolved`, - # which is a WARN -- so a mis-declaration on such a task would be - # announced as merely unread rather than raised as the FAIL it is. - named = base.value if isinstance(base, ast.Subscript) else base - if not isinstance(named, ast.Name): - continue - target = _resolve_intra_package(tree, named.id, path, self.project_root) - if target is None or not target.is_file(): - continue - shared = self._parse_task_module(target, violations) - if shared is None: - continue - parent = _class_named(shared, named.id) - if parent is not None: - inherited += _judgement_references(parent) + inherited = self._inherited_references(cls, tree, path, violations) tiers = ( ("class", _judgement_references(cls)), diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index a4cde348..d583224e 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -169,6 +169,14 @@ from .multi_if import ( MultiIFDataset, MultiIFDatasetSample, ) +from .multipl_e_humaneval import ( + MultiPLEHumanEvalDataset, + MultiPLEHumanEvalDatasetSample, +) +from .multipl_e_mbpp import ( + MultiPLEMbppDataset, + MultiPLEMbppDatasetSample, +) from .openbookqa import ( OpenBookQADataset, OpenBookQADatasetSample, @@ -311,6 +319,10 @@ __all__ = [ "MMMLUDatasetSample", "MultiIFDataset", "MultiIFDatasetSample", + "MultiPLEHumanEvalDataset", + "MultiPLEHumanEvalDatasetSample", + "MultiPLEMbppDataset", + "MultiPLEMbppDatasetSample", "OpenBookQADataset", "OpenBookQADatasetSample", "PlatinumBenchDataset", diff --git a/sieval/datasets/_multipl_e.py b/sieval/datasets/_multipl_e.py new file mode 100644 index 00000000..b41b696f --- /dev/null +++ b/sieval/datasets/_multipl_e.py @@ -0,0 +1,202 @@ +"""Shared loader logic for the MultiPL-E benchmark family. + +MultiPL-E is HumanEval and MBPP machine-translated into other programming +languages, published as one HuggingFace dataset with one config per +(suite, language) pair -- ``humaneval-cpp``, ``mbpp-rs``, and so on. The two +suites share an identical schema and an identical loading contract, so the +processing lives here and the two dataset modules differ only in metadata, the +config prefix, and their language set. + +**A language is named by upstream's registry tag, not by its English name**: +``jl`` not ``julia``, ``ml`` not ``ocaml``, ``rkt`` not ``racket``, ``adb`` not +``ada``. Those tags are the config names, so a wrong one is an unresolvable +config rather than a language that scores badly. + +Python is deliberately absent from both suites: MultiPL-E translates *out of* +Python, so ``humaneval-py`` would be plain HumanEval -- which this repo already +has as ``human_eval_0shot_gen`` / ``_base_gen``. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import TypedDict + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict +from datasets import concatenate_datasets, load_dataset + +from sieval.core.utils.hf import apply_eval_split, ensure_dataset_dict + +# nuprl/MultiPL-E main HEAD (dataset last modified 2025-07-15; static since). +MULTIPL_E_REVISION = "28441b6024e71d4a1c1c0f6bf171c935cd5a43f2" + +# Upstream's registry tags, per suite. These differ: HumanEval was translated to +# Dart and MBPP was not, so the sets are listed separately rather than shared +# with a subtraction -- an asymmetry in upstream's data is not a rule to encode. +HUMANEVAL_LANGUAGES: tuple[str, ...] = ( + "adb", + "clj", + "cpp", + "cs", + "d", + "dart", + "elixir", + "go", + "hs", + "java", + "jl", + "js", + "lua", + "ml", + "php", + "pl", + "r", + "rb", + "rkt", + "rs", + "scala", + "sh", + "swift", + "ts", +) +MBPP_LANGUAGES: tuple[str, ...] = tuple( + lang for lang in HUMANEVAL_LANGUAGES if lang != "dart" +) + + +class MultiPLESampleFields(TypedDict): + """Upstream's eight columns, kept under upstream's own names. + + Each suite declares its own ``TypedDict`` with these fields rather than + sharing this one: the sample type is the reverse-lookup key ``@sieval_task`` + resolves a task's dataset through, so one shared type would make the two + suites indistinguishable to the registry. This exists to state the schema + once, and is not itself registered. + + ``prompt`` is a partial program that stops at the function's opening -- the + model continues it. ``tests`` closes that function and adds the suite's + assertions, so the graded program is ``prompt + completion + tests``. + ``stop_tokens`` truncates a base model's continuation. + + ``original`` is an absolute path on the authors' own cluster and is useless + outside it; it is carried because dropping a column upstream ships is a + divergence with no benefit, not because anything reads it. + """ + + name: str + language: str + prompt: str + doctests: str + original: str + prompt_terminology: str + tests: str + stop_tokens: list[str] + + +def normalize_languages( + languages: list[str] | None, + config: str | None, + *, + suite: str, + available: tuple[str, ...], +) -> tuple[str, ...]: + """Resolve the requested language tags, in upstream's declared order. + + Accepts a bare tag (``cpp``) or a full config name (``humaneval-cpp``) so a + caller who has read the HuggingFace config list can paste from it. A config + naming the *other* suite is rejected rather than silently re-pointed: the + suite is fixed by which dataset class this is, and honouring the config + would load rows the registered task was never bound to. + + Order comes from *available* rather than from the caller, so two runs + requesting the same set in a different order produce the same row order -- + which is what keeps a resume's sample ids stable. + + An EMPTY ``languages`` list is refused rather than read as "all". The two + readings are 24 languages of inference apart, and an empty list reaches here + from a config that computed a selection and came up with nothing far more + often than from an author who meant the whole suite -- which omitting the + argument already says, unambiguously. + """ + if languages is not None and not languages: + raise ValueError( + f"MultiPL-E {suite}: `languages` is an empty list, which names no " + f"language to run. Omit it (or pass `config='all'`) to run all " + f"{len(available)} languages; list registry tags to run a subset." + ) + + requested: list[str] = [] + if config is not None and config != "all": + requested.append(config) + if languages is not None: + requested.extend(languages) + + if not requested: + return available + + prefix = f"{suite}-" + tags: set[str] = set() + unknown: list[str] = [] + for raw in requested: + tag = str(raw).strip() + if tag.startswith(prefix): + tag = tag[len(prefix) :] + elif "-" in tag: + # A config for the other suite, e.g. `mbpp-cpp` asked of HumanEval. + unknown.append(raw) + continue + if tag in available: + tags.add(tag) + else: + unknown.append(raw) + + if unknown: + raise ValueError( + f"Unknown MultiPL-E {suite} language(s): {', '.join(sorted(unknown))}. " + f"Expected upstream registry tags (not English names): " + f"{', '.join(available)}." + ) + return tuple(lang for lang in available if lang in tags) + + +def load_multipl_e( + name_or_path: str, + *, + suite: str, + available: tuple[str, ...], + languages: list[str] | None = None, + config: str | None = None, + eval_split: str | None = None, + **kwargs, +) -> HFDatasetDict: + """Load one or more of *suite*'s language configs as a single DatasetDict. + + All selected languages land in one dataset, the way ``MMMLUDataset`` returns + every locale together rather than one dataset per locale. The rows already + carry a ``language`` column, so a task recovers the per-language breakdown + from the data instead of from how it was loaded. + """ + selected = normalize_languages(languages, config, suite=suite, available=available) + + by_split: dict[str, list[HFDataset]] = {} + for lang in selected: + dataset = load_dataset(name_or_path, f"{suite}-{lang}", **kwargs) + dataset = apply_eval_split(ensure_dataset_dict(dataset), eval_split) + for split, split_dataset in dataset.items(): + if len(split_dataset) > 0: + by_split.setdefault(str(split), []).append(split_dataset) + + processed = HFDatasetDict() + for split, split_datasets in by_split.items(): + processed[split] = ( + split_datasets[0] + if len(split_datasets) == 1 + else concatenate_datasets(split_datasets) + ) + + if not processed or all(len(split) == 0 for split in processed.values()): + raise ValueError( + f"MultiPL-E {suite} loader produced an empty dataset for languages " + f"{', '.join(selected)}; check the language tags, eval_split, or schema." + ) + return processed diff --git a/sieval/datasets/multipl_e_humaneval.py b/sieval/datasets/multipl_e_humaneval.py new file mode 100644 index 00000000..8f783475 --- /dev/null +++ b/sieval/datasets/multipl_e_humaneval.py @@ -0,0 +1,62 @@ +"""MultiPL-E HumanEval loader — HumanEval translated to 24 languages. + +Language selection is a dataset concern: pass ``languages`` (or a single +HuggingFace ``config``) to restrict it. The default is every language upstream +publishes, which is what the benchmark is; whether the deployed code evaluator +can *run* a given language is a separate question, answered by the task at +setup (see ``sieval/tasks/multipl_e/_base.py``). + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import override + +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.datasets import Category, Dataset, Level1Category, sieval_dataset + +from ._multipl_e import ( + HUMANEVAL_LANGUAGES, + MULTIPL_E_REVISION, + MultiPLESampleFields, + load_multipl_e, +) + + +class MultiPLEHumanEvalDatasetSample(MultiPLESampleFields): + """One translated HumanEval problem. Schema documented on the base.""" + + +@sieval_dataset( + name="multipl_e_humaneval", + display_name="MultiPL-E (HumanEval)", + description=( + "HumanEval translated into 24 programming languages; graded by " + "compiling and running the program." + ), + source=f"hf:nuprl/MultiPL-E@{MULTIPL_E_REVISION}", + categories=(Category(Level1Category.CODE, "CodeGeneration"),), + tags=("multilingual", "code-exec"), + license="MIT", +) +class MultiPLEHumanEvalDataset(Dataset[MultiPLEHumanEvalDatasetSample]): + """MultiPL-E's ``humaneval-*`` configs, loaded as one dataset.""" + + @override + def load( + self, + name_or_path: str, + config: str | None = None, + languages: list[str] | None = None, + eval_split: str | None = None, + **kwargs, + ) -> HFDatasetDict: + return load_multipl_e( + name_or_path, + suite="humaneval", + available=HUMANEVAL_LANGUAGES, + languages=languages, + config=config, + eval_split=eval_split, + **kwargs, + ) diff --git a/sieval/datasets/multipl_e_mbpp.py b/sieval/datasets/multipl_e_mbpp.py new file mode 100644 index 00000000..863f04d3 --- /dev/null +++ b/sieval/datasets/multipl_e_mbpp.py @@ -0,0 +1,64 @@ +"""MultiPL-E MBPP loader — MBPP translated to 23 languages. + +Twenty-three, not twenty-four: upstream translated HumanEval to Dart and MBPP +not, so this suite has no ``mbpp-dart`` config. + +Language selection is a dataset concern: pass ``languages`` (or a single +HuggingFace ``config``) to restrict it. The default is every language upstream +publishes; whether the deployed code evaluator can *run* one is answered by the +task at setup (see ``sieval/tasks/multipl_e/_base.py``). + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import override + +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.datasets import Category, Dataset, Level1Category, sieval_dataset + +from ._multipl_e import ( + MBPP_LANGUAGES, + MULTIPL_E_REVISION, + MultiPLESampleFields, + load_multipl_e, +) + + +class MultiPLEMbppDatasetSample(MultiPLESampleFields): + """One translated MBPP problem. Schema documented on the base.""" + + +@sieval_dataset( + name="multipl_e_mbpp", + display_name="MultiPL-E (MBPP)", + description=( + "MBPP translated into 23 programming languages; graded by compiling " + "and running the program." + ), + source=f"hf:nuprl/MultiPL-E@{MULTIPL_E_REVISION}", + categories=(Category(Level1Category.CODE, "CodeGeneration"),), + tags=("multilingual", "code-exec"), + license="MIT", +) +class MultiPLEMbppDataset(Dataset[MultiPLEMbppDatasetSample]): + """MultiPL-E's ``mbpp-*`` configs, loaded as one dataset.""" + + @override + def load( + self, + name_or_path: str, + config: str | None = None, + languages: list[str] | None = None, + eval_split: str | None = None, + **kwargs, + ) -> HFDatasetDict: + return load_multipl_e( + name_or_path, + suite="mbpp", + available=MBPP_LANGUAGES, + languages=languages, + config=config, + eval_split=eval_split, + **kwargs, + ) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index c0d76122..769a99d7 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -972,6 +972,48 @@ "license": "CC-BY-NC-2.0", "checksums": {} }, + { + "name": "multipl_e_humaneval", + "display_name": "MultiPL-E (HumanEval)", + "description": "HumanEval translated into 24 programming languages; graded by compiling and running the program.", + "source": [ + "hf:nuprl/MultiPL-E@28441b6024e71d4a1c1c0f6bf171c935cd5a43f2" + ], + "categories": [ + { + "level1": "Code", + "level2": "CodeGeneration" + } + ], + "tags": [ + "multilingual", + "code-exec" + ], + "deps_group": null, + "license": "MIT", + "checksums": {} + }, + { + "name": "multipl_e_mbpp", + "display_name": "MultiPL-E (MBPP)", + "description": "MBPP translated into 23 programming languages; graded by compiling and running the program.", + "source": [ + "hf:nuprl/MultiPL-E@28441b6024e71d4a1c1c0f6bf171c935cd5a43f2" + ], + "categories": [ + { + "level1": "Code", + "level2": "CodeGeneration" + } + ], + "tags": [ + "multilingual", + "code-exec" + ], + "deps_group": null, + "license": "MIT", + "checksums": {} + }, { "name": "openbookqa", "display_name": "OpenBookQA", @@ -2453,6 +2495,92 @@ "status": "stable", "reference_kind": "value" }, + { + "name": "multipl_e_humaneval_0shot_base_gen", + "display_name": "MultiPL-E HumanEval (0-shot, base generative)", + "description": "HumanEval in 24 languages for base models, graded by compiling and running the program.", + "dataset": "multipl_e_humaneval", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "multilingual", + "code-exec", + "base-model" + ], + "deps_group": null, + "model_type": "gen", + "reference_impl": { + "source": "MultiPL-E", + "url": "https://github.com/nuprl/MultiPL-E/blob/3025a531af7450e7df8b96fe0440e9804480bbad/evaluation/src/main.py", + "notes": "Follows MultiPL-E's base-model path: the prompt is upstream's partial program verbatim, the continuation is truncated at the row's own `stop_tokens` (per language) with upstream's `stop_at_stop_token`, and the graded program is `prompt + completion + \"\\n\" + tests` — `evaluation/src/main.py`'s own assembly, separator included. A program passes iff it builds and exits 0; nothing is parsed or compared. Per-language build and run commands follow `evaluation/src/eval_.py` and live in the code-eval service, not here. DIVERGENCES: (1) execution runs through the SiEval code-eval API rather than upstream's own container, so only languages that service deploys can be graded — validated end-to-end for cpp / js / sh / pl; the task refuses at setup, before spending inference, for any requested language it cannot run. (2) Upstream's four-way failure taxonomy (SyntaxError / AssertionError / ReferenceError / Exception) is reduced to the build-vs-run split the service's one boolean can carry, reported as `n_build_errors` / `n_execution_errors` / `timeouts`; scoring is unaffected. (3) upstream's perl rule that an otherwise-passing run fails when its output contains `ERROR` is implemented in the service, and is unreachable through the shipped test templates — 0 of 161 `humaneval-pl` rows mention `ERROR` and all 161 signal failure with `exit 1` — so it fires only on model stdout. (4) the per-step wall clocks are the service's rather than upstream's: every `safe_subprocess.run` call upstream takes its flat 15s default, the build included, while here an interpreted row (bash, perl) gets 3s to run and a c++ build gets 60s. Both directions are reachable — a program needing between 3s and 15s fails here and passes upstream; a compile upstream abandons at 15s completes here — though the run wall has measured headroom: a pure-bash prime sieve to n=20000 finishes in 0.6s. UNMEASURED: no published-score alignment run has been made, so the score is not yet anchored to a MultiPL-E table — hence `experimental`. Upstream's per-language row counts differ (161 c++, 158 bash), so the per-language `pass@1_` keys are the paper-comparable ones; the headline is a size-weighted pool over whatever languages ran, with `pass@1_macro` beside it. PROTOCOL: upstream publishes pass@1 from 20 completions at temperature 0.2 (`--completion-limit 20`), and pass@10/pass@100 need temperature 0.8 and more samples. This task defaults to n=1, so match upstream with `args.n: 20` plus `infer_args.temperature: 0.2`." + }, + "status": "experimental", + "reference_kind": "procedure" + }, + { + "name": "multipl_e_humaneval_0shot_gen", + "display_name": "MultiPL-E HumanEval (0-shot, generative)", + "description": "HumanEval in 24 languages for chat models, graded by compiling and running the program.", + "dataset": "multipl_e_humaneval", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "multilingual", + "code-exec" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "MultiPL-E", + "url": "https://github.com/nuprl/MultiPL-E/blob/3025a531af7450e7df8b96fe0440e9804480bbad/dataset_builder/chat_completions.py", + "notes": "Follows MultiPL-E's chat path (`dataset_builder/chat_completions.py`). The instruction — repeat the prefix verbatim, complete only the incomplete function, add no tests or usage examples — is upstream's `Completion` signature text carried verbatim, and `postprocess_completion` is reproduced exactly: a reply starting with a fence loses its first and last lines, then the reply is split at `len(prompt)` characters and only the tail is scanned for `stop_tokens`. Critically, the graded program uses upstream's blank prompt — `\"\" + completion + \"\\n\" + tests` — so the MODEL's restatement of the prefix is what compiles. Upstream's comment gives the reason: models paraphrase docstrings and parameter names, and failing a program over that would be a spurious error. Prepending the dataset prompt instead would define the function twice and fail to compile in most of these languages. A further divergence: upstream drives this through DSPy `ChainOfThought`, so its rendered prompt carries DSPy's field markers and a reasoning field; reproducing that would pin the port to a DSPy version rather than to MultiPL-E, so the instruction and all three field descriptions (including the output's `The complete program including the full prefix`, which restates the prefix-repetition the blank-prompt path depends on) are carried in a plain chat prompt and the scaffolding is not. The instruction is a `system` message, where DSPy also puts a signature docstring; a chat template enforcing strict alternation (Mistral's) rejects two consecutive `user` messages anyway. A program passes iff it builds and exits 0; nothing is parsed or compared. Per-language build and run commands follow `evaluation/src/eval_.py` and live in the code-eval service, not here. DIVERGENCES: (1) execution runs through the SiEval code-eval API rather than upstream's own container, so only languages that service deploys can be graded — validated end-to-end for cpp / js / sh / pl; the task refuses at setup, before spending inference, for any requested language it cannot run. (2) Upstream's four-way failure taxonomy (SyntaxError / AssertionError / ReferenceError / Exception) is reduced to the build-vs-run split the service's one boolean can carry, reported as `n_build_errors` / `n_execution_errors` / `timeouts`; scoring is unaffected. (3) upstream's perl rule that an otherwise-passing run fails when its output contains `ERROR` is implemented in the service, and is unreachable through the shipped test templates — 0 of 161 `humaneval-pl` rows mention `ERROR` and all 161 signal failure with `exit 1` — so it fires only on model stdout. (4) the per-step wall clocks are the service's rather than upstream's: every `safe_subprocess.run` call upstream takes its flat 15s default, the build included, while here an interpreted row (bash, perl) gets 3s to run and a c++ build gets 60s. Both directions are reachable — a program needing between 3s and 15s fails here and passes upstream; a compile upstream abandons at 15s completes here — though the run wall has measured headroom: a pure-bash prime sieve to n=20000 finishes in 0.6s. UNMEASURED: no published-score alignment run has been made, so the score is not yet anchored to a MultiPL-E table — hence `experimental`. Upstream's per-language row counts differ (161 c++, 158 bash), so the per-language `pass@1_` keys are the paper-comparable ones; the headline is a size-weighted pool over whatever languages ran, with `pass@1_macro` beside it. PROTOCOL: upstream's chat script takes `--max-completions` with `--temperature 0.2` for pass@1 (20 in its own example). This task defaults to n=1; match upstream with `args.n: 20` plus `infer_args.temperature: 0.2`. No `stop` is sent, deliberately — the reply is asked to contain the prefix, whose text holds this language's stop tokens." + }, + "status": "experimental", + "reference_kind": "procedure" + }, + { + "name": "multipl_e_mbpp_0shot_base_gen", + "display_name": "MultiPL-E MBPP (0-shot, base generative)", + "description": "MBPP in 23 languages for base models, graded by compiling and running the program.", + "dataset": "multipl_e_mbpp", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "multilingual", + "code-exec", + "base-model" + ], + "deps_group": null, + "model_type": "gen", + "reference_impl": { + "source": "MultiPL-E", + "url": "https://github.com/nuprl/MultiPL-E/blob/3025a531af7450e7df8b96fe0440e9804480bbad/evaluation/src/main.py", + "notes": "Follows MultiPL-E's base-model path: the prompt is upstream's partial program verbatim, the continuation is truncated at the row's own `stop_tokens` (per language) with upstream's `stop_at_stop_token`, and the graded program is `prompt + completion + \"\\n\" + tests` — `evaluation/src/main.py`'s own assembly, separator included. A program passes iff it builds and exits 0; nothing is parsed or compared. Per-language build and run commands follow `evaluation/src/eval_.py` and live in the code-eval service, not here. DIVERGENCES: (1) execution runs through the SiEval code-eval API rather than upstream's own container, so only languages that service deploys can be graded — validated end-to-end for cpp / js / sh / pl; the task refuses at setup, before spending inference, for any requested language it cannot run. (2) Upstream's four-way failure taxonomy (SyntaxError / AssertionError / ReferenceError / Exception) is reduced to the build-vs-run split the service's one boolean can carry, reported as `n_build_errors` / `n_execution_errors` / `timeouts`; scoring is unaffected. (3) upstream's perl rule that an otherwise-passing run fails when its output contains `ERROR` is implemented in the service, and is unreachable through the shipped test templates — 0 of 161 `humaneval-pl` rows mention `ERROR` and all 161 signal failure with `exit 1` — so it fires only on model stdout. (4) the per-step wall clocks are the service's rather than upstream's: every `safe_subprocess.run` call upstream takes its flat 15s default, the build included, while here an interpreted row (bash, perl) gets 3s to run and a c++ build gets 60s. Both directions are reachable — a program needing between 3s and 15s fails here and passes upstream; a compile upstream abandons at 15s completes here — though the run wall has measured headroom: a pure-bash prime sieve to n=20000 finishes in 0.6s. UNMEASURED: no published-score alignment run has been made, so the score is not yet anchored to a MultiPL-E table — hence `experimental`. Upstream's per-language row counts differ (161 c++, 158 bash), so the per-language `pass@1_` keys are the paper-comparable ones; the headline is a size-weighted pool over whatever languages ran, with `pass@1_macro` beside it. PROTOCOL: upstream publishes pass@1 from 20 completions at temperature 0.2 (`--completion-limit 20`), and pass@10/pass@100 need temperature 0.8 and more samples. This task defaults to n=1, so match upstream with `args.n: 20` plus `infer_args.temperature: 0.2`. SUITE: MBPP, 23 languages — upstream translated HumanEval to Dart and MBPP not, so there is no `mbpp-dart` config. Upstream's translator also rewrote the word `python` inside MBPP's docstrings along with the code, so prompts read \"Write a cppthon function\" and similar: 132/397 `mbpp-cpp` rows, 131/382 `mbpp-sh`, 131/396 `mbpp-pl`, 132/397 `mbpp-js` — about a third of each language. Carried as-is: the unqualified task tracks upstream, and a prompt fix would be a `datasets/` concern rather than a task one." + }, + "status": "experimental", + "reference_kind": "procedure" + }, + { + "name": "multipl_e_mbpp_0shot_gen", + "display_name": "MultiPL-E MBPP (0-shot, generative)", + "description": "MBPP in 23 languages for chat models, graded by compiling and running the program.", + "dataset": "multipl_e_mbpp", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "multilingual", + "code-exec" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "MultiPL-E", + "url": "https://github.com/nuprl/MultiPL-E/blob/3025a531af7450e7df8b96fe0440e9804480bbad/dataset_builder/chat_completions.py", + "notes": "Follows MultiPL-E's chat path (`dataset_builder/chat_completions.py`). The instruction — repeat the prefix verbatim, complete only the incomplete function, add no tests or usage examples — is upstream's `Completion` signature text carried verbatim, and `postprocess_completion` is reproduced exactly: a reply starting with a fence loses its first and last lines, then the reply is split at `len(prompt)` characters and only the tail is scanned for `stop_tokens`. Critically, the graded program uses upstream's blank prompt — `\"\" + completion + \"\\n\" + tests` — so the MODEL's restatement of the prefix is what compiles. Upstream's comment gives the reason: models paraphrase docstrings and parameter names, and failing a program over that would be a spurious error. Prepending the dataset prompt instead would define the function twice and fail to compile in most of these languages. A further divergence: upstream drives this through DSPy `ChainOfThought`, so its rendered prompt carries DSPy's field markers and a reasoning field; reproducing that would pin the port to a DSPy version rather than to MultiPL-E, so the instruction and all three field descriptions (including the output's `The complete program including the full prefix`, which restates the prefix-repetition the blank-prompt path depends on) are carried in a plain chat prompt and the scaffolding is not. The instruction is a `system` message, where DSPy also puts a signature docstring; a chat template enforcing strict alternation (Mistral's) rejects two consecutive `user` messages anyway. A program passes iff it builds and exits 0; nothing is parsed or compared. Per-language build and run commands follow `evaluation/src/eval_.py` and live in the code-eval service, not here. DIVERGENCES: (1) execution runs through the SiEval code-eval API rather than upstream's own container, so only languages that service deploys can be graded — validated end-to-end for cpp / js / sh / pl; the task refuses at setup, before spending inference, for any requested language it cannot run. (2) Upstream's four-way failure taxonomy (SyntaxError / AssertionError / ReferenceError / Exception) is reduced to the build-vs-run split the service's one boolean can carry, reported as `n_build_errors` / `n_execution_errors` / `timeouts`; scoring is unaffected. (3) upstream's perl rule that an otherwise-passing run fails when its output contains `ERROR` is implemented in the service, and is unreachable through the shipped test templates — 0 of 161 `humaneval-pl` rows mention `ERROR` and all 161 signal failure with `exit 1` — so it fires only on model stdout. (4) the per-step wall clocks are the service's rather than upstream's: every `safe_subprocess.run` call upstream takes its flat 15s default, the build included, while here an interpreted row (bash, perl) gets 3s to run and a c++ build gets 60s. Both directions are reachable — a program needing between 3s and 15s fails here and passes upstream; a compile upstream abandons at 15s completes here — though the run wall has measured headroom: a pure-bash prime sieve to n=20000 finishes in 0.6s. UNMEASURED: no published-score alignment run has been made, so the score is not yet anchored to a MultiPL-E table — hence `experimental`. Upstream's per-language row counts differ (161 c++, 158 bash), so the per-language `pass@1_` keys are the paper-comparable ones; the headline is a size-weighted pool over whatever languages ran, with `pass@1_macro` beside it. PROTOCOL: upstream's chat script takes `--max-completions` with `--temperature 0.2` for pass@1 (20 in its own example). This task defaults to n=1; match upstream with `args.n: 20` plus `infer_args.temperature: 0.2`. No `stop` is sent, deliberately — the reply is asked to contain the prefix, whose text holds this language's stop tokens. SUITE: MBPP, 23 languages — upstream translated HumanEval to Dart and MBPP not, so there is no `mbpp-dart` config. Upstream's translator also rewrote the word `python` inside MBPP's docstrings along with the code, so prompts read \"Write a cppthon function\" and similar: 132/397 `mbpp-cpp` rows, 131/382 `mbpp-sh`, 131/396 `mbpp-pl`, 132/397 `mbpp-js` — about a third of each language. Carried as-is: the unqualified task tracks upstream, and a prompt fix would be a `datasets/` concern rather than a task one." + }, + "status": "experimental", + "reference_kind": "procedure" + }, { "name": "openbookqa_kshot_gen", "display_name": "OpenBookQA (k-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index de0d23a9..6f2e5729 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -157,6 +157,18 @@ from .multi_if_0shot_gen import ( from .multi_if_0shot_gen_fixed import ( MultiIFZeroShotGenFixedTask, ) +from .multipl_e.multipl_e_humaneval_0shot_base_gen import ( + MultiPLEHumanEvalZeroShotBaseGenTask, +) +from .multipl_e.multipl_e_humaneval_0shot_gen import ( + MultiPLEHumanEvalZeroShotGenTask, +) +from .multipl_e.multipl_e_mbpp_0shot_base_gen import ( + MultiPLEMbppZeroShotBaseGenTask, +) +from .multipl_e.multipl_e_mbpp_0shot_gen import ( + MultiPLEMbppZeroShotGenTask, +) from .openbookqa_kshot_gen import ( OpenBookQAFewShotGenTask, ) @@ -265,6 +277,10 @@ __all__ = [ "MMMLUKShotClpTask", "MultiIFZeroShotGenFixedTask", "MultiIFZeroShotGenTask", + "MultiPLEHumanEvalZeroShotBaseGenTask", + "MultiPLEHumanEvalZeroShotGenTask", + "MultiPLEMbppZeroShotBaseGenTask", + "MultiPLEMbppZeroShotGenTask", "OpenBookQAFewShotGenTask", "PlatinumGSM8KZeroShotGenTask", "PlatinumMultiArithZeroShotGenTask", diff --git a/sieval/tasks/multipl_e/__init__.py b/sieval/tasks/multipl_e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/tasks/multipl_e/_base.py b/sieval/tasks/multipl_e/_base.py new file mode 100644 index 00000000..4e201666 --- /dev/null +++ b/sieval/tasks/multipl_e/_base.py @@ -0,0 +1,890 @@ +"""Shared protocol, grading and reporting for the MultiPL-E task family. + +MultiPL-E is HumanEval and MBPP translated into other programming languages. +A problem is a partial program that stops at the function's opening; the model +continues it; the graded program is the prompt, the continuation and the +suite's own assertions, compiled and run in the target language. Nothing is +parsed or compared -- the exit code is the verdict. + +Four tasks come out of two independent choices, and this module holds +everything both axes share. + +**Suite** (``humaneval`` / ``mbpp``) differs only in which dataset the task +binds to. Upstream's two suites have different language sets -- HumanEval was +translated to Dart and MBPP was not. + +**Protocol** is the axis that actually changes behaviour, and upstream +documents two: + +* **Completion** (``_base_gen``) is upstream's primary path, the one the README + leads with. The prompt is the partial program verbatim, the continuation is + truncated at the language's own ``stop_tokens``, and the graded program is + ``prompt + completion``. +* **Chat** (``_gen``) follows ``dataset_builder/chat_completions.py``, which + exists because "the original MultiPL-E prompts are sub-optimal for all chat + models, and are not compatible with chat-only models". It asks for the WHOLE + program back -- prefix repeated verbatim plus the completion -- and then + grades the model's own restatement of the prefix, **discarding the dataset + prompt entirely**. Upstream's comment calls this "Federico's hack" and gives + the reason: models paraphrase docstrings and parameter names, and failing a + program over that would be "a spurious error". + +That second point is the one thing a MultiPL-E port cannot get away with +guessing. Prepending the dataset prompt to a chat reply that already repeats it +duplicates the function definition, which does not compile in most of these +languages -- a whole-benchmark zero that looks like a very bad model. The two +protocols therefore differ in exactly one place, :meth:`prompt_prefix`, and +that place is why they are separate tasks rather than a flag. + +Program assembly is upstream's, separator included +(``evaluation/src/main.py``):: + + program = problem["prompt"] + completion + "\\n" + problem["tests"] + +Execution goes to the SiEval code-eval API rather than to upstream's own +container: sieval already routes every code-exec task through that service, and +a second execution mechanism for one benchmark would be a second thing to +secure. What the service can run is asked at setup, not assumed -- see +:meth:`MultiPLETask.setup`. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import os +import time +from collections import defaultdict +from collections.abc import Sequence +from typing import ClassVar, cast, override + +import httpx +from loguru import logger + +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + JudgementRecord, + PredictionRecord, + PromptRecord, + RolloutJudgement, + Task, + TaskContext, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, + health_metrics, + sampling_report, + ungated_intervals, +) + +# Commit-pinned: `main` moves, and a divergence is argued against the code that +# was actually read. `main.py` is where the graded program is assembled. +MULTIPL_E_COMMIT = "3025a531af7450e7df8b96fe0440e9804480bbad" +MULTIPL_E_UPSTREAM_URL = ( + f"https://github.com/nuprl/MultiPL-E/blob/{MULTIPL_E_COMMIT}/evaluation/src/main.py" +) +CHAT_UPSTREAM_URL = ( + f"https://github.com/nuprl/MultiPL-E/blob/{MULTIPL_E_COMMIT}/" + "dataset_builder/chat_completions.py" +) + +# Client-side walls for one grade request. Both exist so that no grade can wait +# forever: grading is synchronous on one shared event loop, so an unbounded wait +# stalls the session rather than the sample. Sized to sit above the evaluator's +# own server-side budget, whose largest row today is c++'s 15s run plus a +# separate 60s compile. +GRADE_WALL_HEADROOM = 90.0 +DEFAULT_GRADE_WALL = 180.0 + +# `reference_impl.notes`, factored the way the four tasks are: one note per +# protocol, one per suite that has something of its own to say. Held here rather +# than imported leaf-to-leaf, so a shared sentence has one home. +_SHARED_NOTES = ( + "A program passes iff it builds and exits 0; nothing is parsed or compared. " + "Per-language build and run commands follow " + "`evaluation/src/eval_.py` and live in the code-eval service, not " + "here. DIVERGENCES: (1) execution runs through the SiEval code-eval API " + "rather than upstream's own container, so only languages that service " + "deploys can be graded — validated end-to-end for cpp / js / sh / pl; the " + "task refuses at setup, before spending inference, for any requested " + "language it cannot run. (2) Upstream's four-way failure taxonomy " + "(SyntaxError / AssertionError / ReferenceError / Exception) is reduced to " + "the build-vs-run split the service's one boolean can carry, reported as " + "`n_build_errors` / `n_execution_errors` / `timeouts`; scoring is " + "unaffected. (3) upstream's perl rule that an otherwise-passing run fails " + "when its output contains `ERROR` is implemented in the service, and is " + "unreachable through the shipped test templates — 0 of 161 `humaneval-pl` " + "rows mention `ERROR` and all 161 signal failure with `exit 1` — so it " + "fires only on model stdout. (4) the per-step wall clocks are the service's " + "rather than upstream's: every `safe_subprocess.run` call upstream takes its " + "flat 15s default, the build included, while here an interpreted row (bash, " + "perl) gets 3s to run and a c++ build gets 60s. Both directions are " + "reachable — a program needing between 3s and 15s fails here and passes " + "upstream; a compile upstream abandons at 15s completes here — though the " + "run wall has measured headroom: a pure-bash prime sieve to n=20000 " + "finishes in 0.6s. " + "UNMEASURED: no published-score alignment run has been made, so the score " + "is not yet anchored to a MultiPL-E table — hence `experimental`. " + "Upstream's per-language row counts differ (161 c++, 158 bash), so the " + "per-language `pass@1_` keys are the paper-comparable ones; the " + "headline is a size-weighted pool over whatever languages ran, with " + "`pass@1_macro` beside it." +) + +COMPLETION_NOTES = ( + "Follows MultiPL-E's base-model path: the prompt is upstream's partial " + "program verbatim, the continuation is truncated at the row's own " + "`stop_tokens` (per language) with upstream's `stop_at_stop_token`, and " + 'the graded program is `prompt + completion + "\\n" + tests` — ' + "`evaluation/src/main.py`'s own assembly, separator included. " + + _SHARED_NOTES + + " PROTOCOL: upstream publishes pass@1 from 20 completions at temperature " + "0.2 (`--completion-limit 20`), and pass@10/pass@100 need temperature 0.8 " + "and more samples. This task defaults to n=1, so match upstream with " + "`args.n: 20` plus `infer_args.temperature: 0.2`." +) + +CHAT_NOTES = ( + "Follows MultiPL-E's chat path (`dataset_builder/chat_completions.py`). " + "The instruction — repeat the prefix verbatim, complete only the " + "incomplete function, add no tests or usage examples — is upstream's " + "`Completion` signature text carried verbatim, and `postprocess_completion` " + "is reproduced exactly: a reply starting with a fence loses its first and " + "last lines, then the reply is split at `len(prompt)` characters and only " + "the tail is scanned for `stop_tokens`. Critically, the graded program uses " + 'upstream\'s blank prompt — `"" + completion + "\\n" + tests` — so the ' + "MODEL's restatement of the prefix is what compiles. Upstream's comment " + "gives the reason: models paraphrase docstrings and parameter names, and " + "failing a program over that would be a spurious error. Prepending the " + "dataset prompt instead would define the function twice and fail to compile " + "in most of these languages. A further divergence: upstream drives this " + "through DSPy `ChainOfThought`, so its rendered prompt carries DSPy's field " + "markers and a reasoning field; reproducing that would pin the port to a " + "DSPy version rather than to MultiPL-E, so the instruction and all three " + "field descriptions (including the output's `The complete program including " + "the full prefix`, which restates the prefix-repetition the blank-prompt " + "path depends on) are carried in a plain chat prompt and the scaffolding is " + "not. The instruction is a `system` message, where DSPy also puts a " + "signature docstring; a chat template enforcing strict alternation " + "(Mistral's) rejects two consecutive `user` messages anyway. " + + _SHARED_NOTES + + " PROTOCOL: upstream's chat script takes " + "`--max-completions` with `--temperature 0.2` for pass@1 (20 in its own " + "example). This task defaults to n=1; match upstream with `args.n: 20` plus " + "`infer_args.temperature: 0.2`. No `stop` is sent, deliberately — the reply " + "is asked to contain the prefix, whose text holds this language's stop " + "tokens." +) + +MBPP_SUITE_NOTES = ( + " SUITE: MBPP, 23 languages — upstream translated HumanEval to Dart and " + "MBPP not, so there is no `mbpp-dart` config. Upstream's translator also " + "rewrote the word `python` inside MBPP's docstrings along with the code, so " + 'prompts read "Write a cppthon function" and similar: 132/397 `mbpp-cpp` ' + "rows, 131/382 `mbpp-sh`, 131/396 `mbpp-pl`, 132/397 `mbpp-js` — about a " + "third of each language. Carried as-is: the unqualified task tracks " + "upstream, and a prompt fix would be a `datasets/` concern rather than a " + "task one." +) + +# MultiPL-E's registry tag -> the `lang` the code-eval API knows the language +# by. A pure vocabulary map, and complete on purpose: what the DEPLOYED +# evaluator can run is a different question, answered by its own +# `GET /languages` at setup. Keeping the two apart means a language lights up by +# adding a toolchain to the evaluator, with no edit here -- and that a language +# it cannot run is named as such instead of scoring zero. +# +# Names on the right follow the evaluator's existing spelling (`javascript`, +# `typescript`, `python` -- English names, not tags), which is this service's +# vocabulary and only ever this service's: nothing here is passed to upstream's +# `containerized_eval`, whose `EVALUATORS` keys are the tags. Left column +# upstream's, right column the evaluator's, and the map is the seam. +EVALUATOR_LANG_BY_TAG: dict[str, str] = { + "adb": "ada", + "clj": "clojure", + "cpp": "cpp", + "cs": "csharp", + "d": "d", + "dart": "dart", + "elixir": "elixir", + "go": "go", + "hs": "haskell", + "java": "java", + "jl": "julia", + "js": "javascript", + "lua": "lua", + "ml": "ocaml", + "php": "php", + "pl": "perl", + "r": "r", + "rb": "ruby", + "rkt": "racket", + "rs": "rust", + "scala": "scala", + "sh": "bash", + "swift": "swift", + "ts": "typescript", +} + +# Upstream's `Completion` signature docstring and field descriptions, from +# `dataset_builder/chat_completions.py`. Verbatim, because the instruction to +# repeat the prefix is what makes the blank-prompt grading path work: a prompt +# that fails to ask for the prefix produces replies that grade as a bare +# function body with no signature. +CHAT_INSTRUCTION = ( + "I will give you the prefix of an incomplete program that you must " + "complete.\n" + "In your response, do not alter the prefix but repeat it exactly, and when\n" + "you complete the program just complete the incomplete function. Do\n" + "not write additional functions, tests, or usage examples.\n\n" + "You MUST return a full program with the prefix verbatim and the completion." +) + + +def stop_at_stop_token(decoded_string: str, stop_tokens: Sequence[str]) -> str: + """Truncate at the first stop token, upstream's function verbatim. + + ``decoded_string`` must NOT include the prompt -- upstream carries that as a + shouted warning, and the reason is that the prompt contains stop tokens of + its own (``\\n}`` for c++, ``\\nfunction `` for js), so scanning it would + truncate the completion to nothing at position zero. + """ + min_stop_index = len(decoded_string) + for stop_token in stop_tokens: + stop_index = decoded_string.find(stop_token) + if stop_index != -1 and stop_index < min_stop_index: + min_stop_index = stop_index + return decoded_string[:min_stop_index] + + +class MultiPLETask[TSample]( + Task[ + TSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + # `float | str`: the report carries `score_key`, which names a column + # rather than measuring one; `list[float]` carries an interval, and + # `dict[str, str]` the `ci95_units` map naming each interval's unit. + dict[str, float | str | list[float] | dict[str, str]], + ] +): + """Grading, reporting and the setup probe. Protocol lives in the subclasses. + + Leaves supply the sample type; subclasses supply :attr:`suite`, + :attr:`eval_source` and the protocol methods. + """ + + # `humaneval` / `mbpp` -- only used in messages; the dataset decides the rows. + suite: ClassVar[str] = "" + # The code-eval API's direct-run alias for this suite. Both aliases reach + # the same code path; sending the matching one keeps the service's log + # readable rather than attributing every MBPP run to human-eval. + eval_source: ClassVar[str] = "" + + def __init__( + self, + dataset, + model, + name: str | None = None, + k: int = 1, + n: int = 1, + max_concurrency: int = 4, + # Upstream's own per-program wall (`libeval.run_without_exn` uses 5s, + # `safe_subprocess.run` 15s). Left at the evaluator's per-language + # default when None, which is where a compiled language's larger budget + # is declared -- overriding it here with one number would charge c++ the + # same wall as bash. + timeout: float | None = None, + ): + super().__init__(dataset=dataset, model=model, name=name) + if k > n: + raise ValueError( + f"pass@{k} needs at least {k} sample(s) per problem, got n={n}. " + "Raise the task arg `n` (tasks..args.n) to at least k — " + "setting `n` on the model is silently overridden call-time." + ) + self._k = k + self._n = n + self._timeout = timeout + # Filled on first use, off the live test set -- see `_language_column`. + self._languages_by_sample_id: list[str] | None = None + self._code_eval_api = os.getenv( + "SIEVAL_CODE_EVAL_API", "http://localhost:11451/evaluations" + ) + self._http_client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=max_concurrency) + ) + + # ---- protocol hooks ---------------------------------------------------- + + def prompt_prefix(self, raw) -> str: + """What the graded program starts with, before the model's text. + + The whole difference between the two protocols. Completion returns the + dataset prompt; chat returns ``""``, because the model was asked to + repeat the prefix itself and upstream grades that copy. + """ + raise NotImplementedError + + # ---- shared lifecycle -------------------------------------------------- + + @override + async def setup(self) -> None: + """Refuse the run if the evaluator cannot execute a requested language. + + Checked BEFORE any inference, because the alternative is silent and + expensive: an unsupported language returns a clean failed verdict per + sample, so the run completes, costs a full generation budget, and + reports ``pass@1 = 0`` with no errors — indistinguishable from a model + that simply cannot write Racket. Naming the gap up front is the only + point at which it is cheap. + """ + requested = self._requested_languages() + unmapped = sorted(set(requested) - set(EVALUATOR_LANG_BY_TAG)) + if unmapped: + raise ValueError( + f"MultiPL-E {self.suite}: no code-eval language is mapped for " + f"tag(s) {', '.join(unmapped)}. Add them to " + f"EVALUATOR_LANG_BY_TAG." + ) + + advertised = await self._advertised_languages() + missing = sorted( + {tag for tag in requested if EVALUATOR_LANG_BY_TAG[tag] not in advertised} + ) + if missing: + runnable = sorted( + tag for tag, lang in EVALUATOR_LANG_BY_TAG.items() if lang in advertised + ) + raise ValueError( + f"MultiPL-E {self.suite}: the code-eval service at " + f"{self._code_eval_api} cannot run " + f"{', '.join(missing)} " + f"(needs {', '.join(EVALUATOR_LANG_BY_TAG[t] for t in missing)}). " + f"Deploy those toolchains, or restrict the run with the dataset " + f"arg `languages`. Runnable there now: " + f"{', '.join(runnable) or '(none)'}." + ) + + async def _advertised_languages(self) -> frozenset[str]: + """The `lang` values the deployed evaluator says it accepts. + + A service without the endpoint is refused rather than assumed capable: + it predates table-driven languages, so it cannot run any of the ones + this benchmark needs beyond js/ts, and treating "cannot ask" as "can + run" is what turns a deployment gap into a run of zeros. + """ + # `rstrip` first: a trailing slash would otherwise make `rsplit` drop the + # empty last segment instead of `evaluations`, probing + # `.../evaluations/languages` -- a 404, reported as "this deployment + # predates table-driven languages" when the deployment is in fact fine. + url = self._code_eval_api.rstrip("/").rsplit("/", 1)[0] + "/languages" + try: + resp = await self._http_client.get(url, timeout=10.0) + except Exception as e: + raise RuntimeError( + f"MultiPL-E {self.suite}: cannot reach the code-eval service at " + f"{url} ([{type(e).__name__}] {e}). It must be running before a " + f"run starts — grading a MultiPL-E program needs its language's " + f"toolchain, so there is nothing to fall back to." + ) from e + if resp.status_code == 404: + raise RuntimeError( + f"MultiPL-E {self.suite}: the code-eval service at {url} has no " + f"`/languages` endpoint, so it predates table-driven language " + f"support and cannot run this benchmark. Redeploy from " + f"`vendor/code-evaluator` (see docker/Dockerfile.multipl-e)." + ) + # Wrapped like the connection failure above: a 5xx here is a deployment + # answer, not a task bug, and bare it was the one branch of this probe + # that surfaced as an `HTTPStatusError` with no reading. + try: + resp.raise_for_status() + except httpx.HTTPStatusError as e: + raise RuntimeError( + f"MultiPL-E {self.suite}: the code-eval service at {url} " + f"answered HTTP {resp.status_code} for its own capability " + f"probe, so what it can run is unknown and a run would be " + f"guessing. Check the service's logs before starting one." + ) from e + payload = resp.json() + return frozenset(payload.get("data") or ()) + + def _language_column(self) -> list[str]: + """The whole ``language`` column, read once and cached. + + One column read rather than a row read per sample: an Arrow row access + pulls the row group its column chunk lives in, so placing every sample + one at a time would re-pay that for each of them, twice over in + ``_per_language_metrics``. The test set is fixed for a run, so caching + it cannot go stale. + """ + if self._languages_by_sample_id is None: + test_set = self._dataset.test_set + if not test_set or "language" not in test_set.column_names: + raise ValueError( + f"MultiPL-E {self.suite}: the test set carries no `language` " + f"column; grading cannot pick a toolchain per problem." + ) + self._languages_by_sample_id = [ + str(value) for value in test_set["language"] + ] + return self._languages_by_sample_id + + def _requested_languages(self) -> tuple[str, ...]: + """Language tags present in the test set, in first-seen order.""" + seen: dict[str, None] = {} + for value in self._language_column(): + seen.setdefault(value, None) + return tuple(seen) + + def _language_of(self, ctx: TaskContext) -> str: + """The language of *ctx*'s problem, read off the live dataset. + + Read from the dataset rather than from the persisted judgement, for the + reason :meth:`Task.problem_groups` gives: a resume rebuilds every + context through ``make_context`` off the same index, so this resolves + identically fresh and resumed — and it is the only route that also works + for a FAILED sample, which has no judgement to read a language out of. + Without that, a failure could not be attributed to a language and the + per-language denominators would silently disagree with the headline's. + """ + column = self._language_column() + sample_id = ctx.sample_id + if isinstance(sample_id, int) and 0 <= sample_id < len(column): + return column[sample_id] + raise ValueError( + f"MultiPL-E {self.suite}: sample {sample_id!r} cannot be placed in " + f"the test set, so its language is unknown. A per-language rate " + f"computed over an unplaceable sample would be wrong in a way the " + f"report cannot show." + ) + + def _grade_wall(self) -> float: + """The client-side wall for one grade request. Never unbounded. + + The evaluator enforces its own budget server-side, so this only has to + sit ABOVE it — a client wall below it would report a network timeout for + a program the service was still compiling. But it must EXIST: grading is + synchronous on one shared event loop, so a service that hangs or a + connection that stalls would take the whole session down rather than one + sample. + + :data:`GRADE_WALL_HEADROOM` is added to a caller-supplied budget; + :data:`DEFAULT_GRADE_WALL` covers the case where the budget is the + evaluator's own per-language default, which this side does not know. + """ + if self._timeout is None: + return DEFAULT_GRADE_WALL + return self._timeout + GRADE_WALL_HEADROOM + + @override + async def shutdown(self) -> None: + await self._http_client.aclose() + + # ---- grading ----------------------------------------------------------- + + @override + async def feedback(self, post, ctx): + raw = ctx.raw_sample + language = str(raw["language"]) + lang = EVALUATOR_LANG_BY_TAG[language] + prefix = self.prompt_prefix(raw) + + rollouts: list[RolloutJudgement] = [] + for rollout in post["rollouts"]: + idx = rollout["index"] + # An unextractable completion is None here but "" on the wire, so + # the evaluator still runs the program and reports a real build + # failure rather than the run skipping a verdict. + program = prefix + (rollout.get("prediction") or "") + "\n" + raw["tests"] + payload = { + "uuid": f"{idx}-{time.perf_counter_ns()}", + "source": self.eval_source, + "lang": lang, + "code": program, + } + if self._timeout is not None: + payload["timeout"] = self._timeout + try: + resp = await self._http_client.post( + self._code_eval_api, + json=payload, + timeout=self._grade_wall(), + ) + resp.raise_for_status() + res = resp.json() + data = res["data"] or {} + rollouts.append( + build_rollout_judgement( + idx, + res["status"], + extra={ + "msg": res["msg"], + # Persisted so a per-language rate can be recomputed + # from the shard files alone, without the dataset. + "language": language, + "n_cases": data.get("n_cases"), + "n_passed": data.get("n_passed"), + "resources": { + key: value + for key, value in data.items() + if key not in ("n_cases", "n_passed") + }, + }, + ) + ) + except Exception as e: + logger.warning( + "Evaluation error for sample {} ({}): [{}] {}", + idx, + language, + type(e).__name__, + e, + ) + raise e + + return True, build_judgement_record( + None, # the reference is the test suite, not a value + rollouts, + extra={"name": str(raw["name"]), "language": language}, + ) + + # ---- reporting --------------------------------------------------------- + + @override + async def report(self, finals, fails): + total = len(finals) + len(fails) + if total == 0: + # Declarations and COUNTS both belong on this path: a key the full + # report always writes but this one omits makes the schema depend on + # whether any sample survived, which reads as a missing measurement + # rather than an empty one. No interval pair, though -- there is + # nothing to estimate, and a zeroed population would read as + # measured. + return { + "score": 0.0, + "pass@1": 0.0, + "fails": 0.0, + "timeouts": 0.0, + "n_build_errors": 0.0, + "n_execution_errors": 0.0, + "n_languages": 0.0, + "pass@1_macro": 0.0, + SCORE_KEY_FIELD: "pass@1", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + # Merged with `|`, never spread with `**` inside the literal: a + # `**` leaves a key unnameable to `check_report_declarations`, + # which then stops verifying this report's `score_key` while + # still reporting PASS. A merge it can follow into `metrics.py` + # keeps the check. + } | health_metrics([]) + + buckets = self._failure_buckets(finals) + # `votes=False`: two correct programs are not one answer, so there is + # nothing well-defined to take a majority over (RFC #74). + rolled = sampling_report( + finals, + n=self._n, + k=self._k, + denominator=total, + votes=False, + score_key="pass@1", + grouping=self.problem_groups(finals), + ) + # Read back out of the shared block, so `score` cannot drift from it. + pass_at_1 = rolled["pass@1"] + metrics: dict[str, float | str | list[float] | dict[str, str]] = { + "score": pass_at_1, + "fails": float(len(fails)), + "pass@1": pass_at_1, + SCORE_KEY_FIELD: "pass@1", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + metrics |= buckets + metrics |= self._per_language_metrics(finals, fails) + # Outside the n>1 gate, because the metrics they bracket are: `pass@1` + # is published at every budget, and so is the headline copied from it. + metrics |= ungated_intervals(rolled, metrics=("score", "pass@1")) + if self._n > 1: + # At n=1 the rest only restates `pass@1`. + metrics.update(rolled) + # Outside the gate: extraction health is a fact about the parser, not + # about the draw, and n=1 is where a stopped extractor hides longest. + return metrics | health_metrics(finals) + + def _failure_buckets(self, finals) -> dict[str, float]: + """Split failed rollouts by the stage that failed. + + Upstream sorts a failure four ways (``SyntaxError`` / ``AssertionError`` + / ``ReferenceError`` / ``Exception``); the code-eval API answers with one + boolean and a message, so what survives is the split a caller cannot + reconstruct from the verdict — build versus run. Read off the message + prefixes the service documents, the same way the HumanEval tasks read + theirs for `timeouts`. + + ``n_execution_errors`` keeps the name it has in every other task that + executes a prediction; the build bucket is new because no other task has + a compile step to lose a program in. + + Matched on the message's PREFIX, not by substring. The evaluator's + messages are a small closed vocabulary it documents, but their tails + carry the compiler's and the program's own output — so a build error + whose diagnostic happens to contain the word "timeout" (``error: no + member named 'timeout'``) reads as a timed-out run under a substring + test, in the one bucket that suggests the model's program was slow + rather than broken. A build that exceeds ITS wall lands in the build + bucket rather than under ``timeouts``: the run never started, and + build-versus-run is the split these three keys exist to carry. + """ + timeouts = build_errors = execution_errors = 0 + for final in finals: + feedback = final.feedback_result + if feedback is None: + continue + for rollout in feedback["rollouts"]: + if rollout["correct"]: + continue + # A null msg from the evaluator is absent on disk -- default it. + msg = (rollout["extra"].get("msg") or "").lower() + if msg.startswith(("failed: build timeout", "failed [build exit")): + build_errors += 1 + elif msg.startswith("failed: timeout"): + timeouts += 1 + else: + execution_errors += 1 + return { + "timeouts": float(timeouts), + "n_build_errors": float(build_errors), + "n_execution_errors": float(execution_errors), + } + + def _per_language_metrics(self, finals, fails) -> dict[str, float]: + """``pass@1`` per language, plus the unweighted mean over languages. + + MultiPL-E publishes one column per language and no single number, so the + per-language keys are the comparable ones and the headline is a POOLED + micro-average over whatever languages the run covered. That pooling is + size-weighted, and upstream's row counts differ by language (161 for + c++, 158 for bash), so ``pass@1_macro`` is published beside it rather + than left for a reader to reconstruct — they are different numbers and + only the per-language columns are comparable with a paper. + + Each language's rate comes from :func:`sampling_report` over that + language's own samples, not from a hand-rolled mean: the estimator is + then the headline's by construction, so a macro cannot drift from the + pooled number for reasons other than weighting. + + No intervals on these keys. Each language is its own axis with its own + population, and one ``n_problems`` cannot carry all of them — the same + reason ``mmmlu_kshot_clp`` leaves its per-locale breakdown bare. + """ + by_language: dict[str, list] = defaultdict(list) + for final in finals: + by_language[self._language_of(final)].append(final) + requested: dict[str, int] = defaultdict(int) + for ctx in (*finals, *fails): + requested[self._language_of(ctx)] += 1 + + metrics: dict[str, float] = {"n_languages": float(len(requested))} + rates: list[float] = [] + for language in sorted(requested): + language_finals = by_language.get(language, []) + metrics[f"n_problems_{language}"] = float(requested[language]) + if not language_finals: + # Every sample of this language failed the pipeline. Under + # DENOMINATOR_REQUESTED that is a real 0, not a missing cell. + metrics[f"pass@1_{language}"] = 0.0 + rates.append(0.0) + continue + block = sampling_report( + language_finals, + n=self._n, + k=self._k, + denominator=requested[language], + votes=False, + grouping=self.problem_groups(language_finals), + ) + # `sampling_report`'s value type spans intervals and the units map, + # but `pass@1` is documented as always present and always a bare + # rate -- which is why it is the key a headline is read back from. + rate = cast(float, block["pass@1"]) + metrics[f"pass@1_{language}"] = rate + rates.append(rate) + + metrics["pass@1_macro"] = sum(rates) / len(rates) if rates else 0.0 + return metrics + + +class MultiPLECompletionTask[TSample](MultiPLETask[TSample]): + """Upstream's primary protocol: the model continues a partial program. + + The prompt is the partial program verbatim -- no instruction, no wrapper -- + so this needs a base (completion) model. Decoding params come from the + model's configured args or per-task ``infer_args``; the task owns only the + per-sample stop tokens and ``n``. + """ + + @override + async def preprocess(self, raw, ctx): + return build_prompt_record( + raw["prompt"], + # No `reference`: the ground truth is a test suite, not a value. It + # is described at judgement time instead. + extra={"name": str(raw["name"]), "language": str(raw["language"])}, + ) + + @override + async def infer(self, pre, ctx): + # Stop tokens are per LANGUAGE, so they come off the sample rather than + # from a task constant: `\n}` ends a c++ function and would never fire + # in Python, and `\nfunction ` is js-only. + return await self.model.agenerate( + pre["prompt"], + n=self._n, + stop=list(ctx.raw_sample["stop_tokens"]), + ) + + @override + async def postprocess(self, inf, ctx): + stop_tokens = list(ctx.raw_sample["stop_tokens"]) + predictions: list[str | None] = [] + for text in inf.texts: + # Truncated here as well as server-side: upstream truncates its own + # decoded output, and a server that returns the stop sequence (or + # honours none) would otherwise leave a second function definition + # in the program. + completion = stop_at_stop_token(text, stop_tokens) + # A blank completion normalizes to None so `extracted` stays a real + # signal -- the program is still built and still graded. + predictions.append(completion or None) + return build_prediction_record(predictions) + + @override + def prompt_prefix(self, raw) -> str: + return str(raw["prompt"]) + + +class MultiPLEChatTask[TSample](MultiPLETask[TSample]): + """Upstream's chat protocol, including the blank-prompt grading path. + + The model is asked for the whole program -- prefix repeated verbatim plus + the completion -- and the graded program is built from the model's own copy + of the prefix, with the dataset prompt discarded. See the module docstring + for why that is upstream's rule rather than a shortcut. + + One documented divergence: upstream drives this through DSPy's + ``ChainOfThought``, whose rendered prompt carries DSPy's own field markers + and a reasoning field. Reproducing that byte-for-byte would pin the port to + a DSPy version rather than to MultiPL-E, so all three field descriptions and + the instruction are carried verbatim in a plain chat prompt and the + scaffolding is not. The instruction is the load-bearing part -- it is what + makes the reply repeat the prefix for the blank-prompt path to grade -- and + upstream's output-field description restates it, so that is carried too. + + The instruction rides in a ``system`` message, which is where DSPy puts a + signature's docstring, so the role split is upstream's. It is also required: + a chat template enforcing strict alternation (Mistral's) rejects two + consecutive ``user`` messages, and ``normalize_chat_input`` does not merge + them. + """ + + @override + async def preprocess(self, raw, ctx): + language = str(raw["language"]) + return build_prompt_record( + [ + {"role": "system", "content": CHAT_INSTRUCTION}, + { + "role": "user", + "content": ( + f"The programming language of the program to complete: " + f"{language}\n\n" + f"The prefix of the program to complete:\n{raw['prompt']}\n\n" + f"Return the complete program including the full prefix." + ), + }, + ], + # No `reference`: the ground truth is a test suite, not a value. + extra={"name": str(raw["name"]), "language": language}, + ) + + @override + async def infer(self, pre, ctx): + # No `stop`: the reply is asked to contain the prefix, whose own text + # holds this language's stop tokens, so passing them would cut the reply + # off inside the part it was told to repeat. Truncation happens in + # postprocess, past the prefix, exactly as upstream does it. + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + prompt = str(ctx.raw_sample["prompt"]) + stop_tokens = list(ctx.raw_sample["stop_tokens"]) + predictions: list[str | None] = [] + for text in inf.texts: + predictions.append( + _postprocess_chat_completion(text, prompt, stop_tokens) or None + ) + return build_prediction_record(predictions) + + @override + def prompt_prefix(self, raw) -> str: + # Blank, and this is the whole point: the reply already carries a copy + # of the prefix, and upstream grades that copy rather than the dataset's + # -- so prepending the dataset prompt here would define the function + # twice. + return "" + + +def _postprocess_chat_completion( + completion: str, prompt: str, stop_tokens: Sequence[str] +) -> str: + """``chat_completions.postprocess_completion``, upstream's logic verbatim. + + Two steps, and the second is stranger than it looks: + + 1. A reply that STARTS with a fence is assumed to be one whole Markdown + block, and its first and last lines are dropped. Upstream does not + search for a fence anywhere else, and does not check that the last line + closes one -- so a reply that opens with a fence and then keeps talking + loses its final line of code. Kept as-is; it is upstream's rule. + 2. The reply is split at ``len(prompt)`` CHARACTERS and only the tail is + scanned for stop tokens. The split point is a heuristic about the + *reply*, not a match against the prompt, and the two halves are + immediately re-joined -- so the only thing it decides is where stop-token + scanning may begin. That is what keeps a stop token belonging to the + repeated prefix from truncating the whole program to nothing. + """ + if completion.startswith("```"): + lines = completion.split("\n") + lines = lines[1:-1] + completion = "\n".join(lines) + + head = completion[: len(prompt)] + tail = stop_at_stop_token(completion[len(prompt) :], stop_tokens) + return head + tail + + +__all__ = [ + "CHAT_INSTRUCTION", + "CHAT_NOTES", + "CHAT_UPSTREAM_URL", + "COMPLETION_NOTES", + "DEFAULT_GRADE_WALL", + "EVALUATOR_LANG_BY_TAG", + "GRADE_WALL_HEADROOM", + "MBPP_SUITE_NOTES", + "MULTIPL_E_COMMIT", + "MULTIPL_E_UPSTREAM_URL", + "MultiPLEChatTask", + "MultiPLECompletionTask", + "MultiPLETask", + "stop_at_stop_token", +] diff --git a/sieval/tasks/multipl_e/multipl_e_humaneval_0shot_base_gen.py b/sieval/tasks/multipl_e/multipl_e_humaneval_0shot_base_gen.py new file mode 100644 index 00000000..e9b1ec16 --- /dev/null +++ b/sieval/tasks/multipl_e/multipl_e_humaneval_0shot_base_gen.py @@ -0,0 +1,42 @@ +"""MultiPL-E HumanEval, base-model completion protocol. + +Upstream's primary path: the prompt is a partial program, the model continues +it, and the completed program is compiled and run against the suite's +assertions. Protocol, grading and the traps are documented once in +:mod:`sieval.tasks.multipl_e._base`. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import ClassVar + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import MultiPLEHumanEvalDatasetSample + +from ._base import COMPLETION_NOTES, MULTIPL_E_UPSTREAM_URL, MultiPLECompletionTask + + +@sieval_task( + name="multipl_e_humaneval_0shot_base_gen", + display_name="MultiPL-E HumanEval (0-shot, base generative)", + description=( + "HumanEval in 24 languages for base models, graded by compiling and " + "running the program." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("multilingual", "code-exec", "base-model"), + model_type="gen", + status="experimental", + reference_kind="procedure", + reference_impl=ReferenceImpl( + source="MultiPL-E", + url=MULTIPL_E_UPSTREAM_URL, + notes=COMPLETION_NOTES, + ), +) +class MultiPLEHumanEvalZeroShotBaseGenTask( + MultiPLECompletionTask[MultiPLEHumanEvalDatasetSample] +): + suite: ClassVar[str] = "humaneval" + eval_source: ClassVar[str] = "human-eval" diff --git a/sieval/tasks/multipl_e/multipl_e_humaneval_0shot_gen.py b/sieval/tasks/multipl_e/multipl_e_humaneval_0shot_gen.py new file mode 100644 index 00000000..9aa34bd2 --- /dev/null +++ b/sieval/tasks/multipl_e/multipl_e_humaneval_0shot_gen.py @@ -0,0 +1,43 @@ +"""MultiPL-E HumanEval, chat-model protocol. + +Follows upstream's ``dataset_builder/chat_completions.py``, which exists +because the original completion prompts "are not compatible with chat-only +models". The model is asked for the whole program back and the graded program +is built from ITS copy of the prefix, not the dataset's. Protocol, grading and +the traps are documented once in :mod:`sieval.tasks.multipl_e._base`. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import ClassVar + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import MultiPLEHumanEvalDatasetSample + +from ._base import CHAT_NOTES, CHAT_UPSTREAM_URL, MultiPLEChatTask + + +@sieval_task( + name="multipl_e_humaneval_0shot_gen", + display_name="MultiPL-E HumanEval (0-shot, generative)", + description=( + "HumanEval in 24 languages for chat models, graded by compiling and " + "running the program." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("multilingual", "code-exec"), + model_type="chat", + status="experimental", + reference_kind="procedure", + reference_impl=ReferenceImpl( + source="MultiPL-E", + url=CHAT_UPSTREAM_URL, + notes=CHAT_NOTES, + ), +) +class MultiPLEHumanEvalZeroShotGenTask( + MultiPLEChatTask[MultiPLEHumanEvalDatasetSample] +): + suite: ClassVar[str] = "humaneval" + eval_source: ClassVar[str] = "human-eval" diff --git a/sieval/tasks/multipl_e/multipl_e_mbpp_0shot_base_gen.py b/sieval/tasks/multipl_e/multipl_e_mbpp_0shot_base_gen.py new file mode 100644 index 00000000..ab44760d --- /dev/null +++ b/sieval/tasks/multipl_e/multipl_e_mbpp_0shot_base_gen.py @@ -0,0 +1,46 @@ +"""MultiPL-E MBPP, base-model completion protocol. + +Same protocol as the HumanEval sibling over MBPP's problems, in the 23 +languages upstream translated MBPP to (no Dart). Protocol, grading and the +traps are documented once in :mod:`sieval.tasks.multipl_e._base`. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import ClassVar + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import MultiPLEMbppDatasetSample + +from ._base import ( + COMPLETION_NOTES, + MBPP_SUITE_NOTES, + MULTIPL_E_UPSTREAM_URL, + MultiPLECompletionTask, +) + + +@sieval_task( + name="multipl_e_mbpp_0shot_base_gen", + display_name="MultiPL-E MBPP (0-shot, base generative)", + description=( + "MBPP in 23 languages for base models, graded by compiling and " + "running the program." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("multilingual", "code-exec", "base-model"), + model_type="gen", + status="experimental", + reference_kind="procedure", + reference_impl=ReferenceImpl( + source="MultiPL-E", + url=MULTIPL_E_UPSTREAM_URL, + notes=COMPLETION_NOTES + MBPP_SUITE_NOTES, + ), +) +class MultiPLEMbppZeroShotBaseGenTask( + MultiPLECompletionTask[MultiPLEMbppDatasetSample] +): + suite: ClassVar[str] = "mbpp" + eval_source: ClassVar[str] = "mbpp" diff --git a/sieval/tasks/multipl_e/multipl_e_mbpp_0shot_gen.py b/sieval/tasks/multipl_e/multipl_e_mbpp_0shot_gen.py new file mode 100644 index 00000000..2aa5335d --- /dev/null +++ b/sieval/tasks/multipl_e/multipl_e_mbpp_0shot_gen.py @@ -0,0 +1,45 @@ +"""MultiPL-E MBPP, chat-model protocol. + +Same protocol as the HumanEval chat sibling over MBPP's problems, in the 23 +languages upstream translated MBPP to (no Dart). The graded program is built +from the model's own copy of the prefix, not the dataset's — see +:mod:`sieval.tasks.multipl_e._base` for why that is upstream's rule. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from typing import ClassVar + +from sieval.core.tasks import EvalMode, ReferenceImpl, sieval_task +from sieval.datasets import MultiPLEMbppDatasetSample + +from ._base import ( + CHAT_NOTES, + CHAT_UPSTREAM_URL, + MBPP_SUITE_NOTES, + MultiPLEChatTask, +) + + +@sieval_task( + name="multipl_e_mbpp_0shot_gen", + display_name="MultiPL-E MBPP (0-shot, generative)", + description=( + "MBPP in 23 languages for chat models, graded by compiling and " + "running the program." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("multilingual", "code-exec"), + model_type="chat", + status="experimental", + reference_kind="procedure", + reference_impl=ReferenceImpl( + source="MultiPL-E", + url=CHAT_UPSTREAM_URL, + notes=CHAT_NOTES + MBPP_SUITE_NOTES, + ), +) +class MultiPLEMbppZeroShotGenTask(MultiPLEChatTask[MultiPLEMbppDatasetSample]): + suite: ClassVar[str] = "mbpp" + eval_source: ClassVar[str] = "mbpp" diff --git a/tests/unit/datasets/test__multipl_e.py b/tests/unit/datasets/test__multipl_e.py new file mode 100644 index 00000000..4c6bf375 --- /dev/null +++ b/tests/unit/datasets/test__multipl_e.py @@ -0,0 +1,124 @@ +"""Unit tests for the shared MultiPL-E dataset loader. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest + +from sieval.datasets._multipl_e import ( + HUMANEVAL_LANGUAGES, + MBPP_LANGUAGES, + normalize_languages, +) + + +def _he(languages=None, config=None): + return normalize_languages( + languages, config, suite="humaneval", available=HUMANEVAL_LANGUAGES + ) + + +def _mbpp(languages=None, config=None): + return normalize_languages( + languages, config, suite="mbpp", available=MBPP_LANGUAGES + ) + + +def test_suites_differ_only_by_dart(): + # Upstream translated HumanEval to Dart and MBPP not. Asserted because the + # two tuples are written out separately, and a copy-paste that unified them + # would make `mbpp-dart` look loadable. + assert set(HUMANEVAL_LANGUAGES) - set(MBPP_LANGUAGES) == {"dart"} + assert set(MBPP_LANGUAGES) - set(HUMANEVAL_LANGUAGES) == set() + assert len(HUMANEVAL_LANGUAGES) == 24 + assert len(MBPP_LANGUAGES) == 23 + + +def test_tags_are_upstream_registry_names_not_english(): + # The tags ARE the HuggingFace config names, so an English name is an + # unresolvable config rather than a language that scores badly. + for tag in ("jl", "ml", "rkt", "adb", "rb", "hs", "cs", "sh", "pl"): + assert tag in HUMANEVAL_LANGUAGES + for english in ("julia", "ocaml", "racket", "ada", "ruby", "haskell", "bash"): + assert english not in HUMANEVAL_LANGUAGES + + +def test_no_python_config(): + # MultiPL-E translates OUT of Python, so a `py` config would be plain + # HumanEval -- which this repo already has as its own task. + assert "py" not in HUMANEVAL_LANGUAGES + assert "python" not in HUMANEVAL_LANGUAGES + assert "py" not in MBPP_LANGUAGES + + +def test_default_selects_every_language(): + assert _he() == HUMANEVAL_LANGUAGES + assert _mbpp() == MBPP_LANGUAGES + # `config="all"` is MMMLU's spelling for the same thing. + assert _he(config="all") == HUMANEVAL_LANGUAGES + + +def test_an_empty_language_list_is_refused_rather_than_read_as_all(): + """Omitting the argument means "all"; an empty list means a config bug. + + The two readings are 24 languages of inference apart, so the silent one is + the expensive one — and a caller that computed a selection and came up with + nothing is far likelier than one who spelled "everything" as `[]`. + """ + for select in (_he, _mbpp): + with pytest.raises(ValueError, match="empty list"): + select([]) + + +def test_bare_tags_and_full_config_names_both_resolve(): + assert _he(["cpp", "js"]) == ("cpp", "js") + assert _he(config="humaneval-rs") == ("rs",) + assert _mbpp(config="mbpp-cpp") == ("cpp",) + + +def test_order_follows_upstream_not_the_caller(): + # Row order decides sample ids, so a resume needs the same order from the + # same set however the caller happened to spell it. + assert ( + _he(["ts", "cpp", "adb"]) + == _he(["adb", "ts", "cpp"]) + == ( + "adb", + "cpp", + "ts", + ) + ) + + +def test_duplicates_collapse(): + assert _he(["cpp", "cpp", "js"]) == ("cpp", "js") + + +def test_english_language_name_is_rejected(): + with pytest.raises(ValueError, match="julia"): + _he(["julia"]) + + +def test_cross_suite_config_is_rejected(): + # `mbpp-cpp` asked of HumanEval would load rows the registered task is not + # bound to, so it is refused rather than silently re-pointed at `cpp`. + with pytest.raises(ValueError, match="mbpp-cpp"): + _he(["mbpp-cpp"]) + with pytest.raises(ValueError, match="humaneval-cpp"): + _mbpp(["humaneval-cpp"]) + + +def test_dart_is_rejected_for_mbpp_only(): + assert _he(["dart"]) == ("dart",) + with pytest.raises(ValueError, match="dart"): + _mbpp(["dart"]) + + +def test_error_names_the_expected_vocabulary(): + # The message has to teach the tag/English-name distinction, since that is + # the mistake it exists to catch. + with pytest.raises(ValueError) as excinfo: + _he(["racket"]) + message = str(excinfo.value) + assert "registry tags" in message + assert "rkt" in message diff --git a/tests/unit/scripts/test_check_preflight.py b/tests/unit/scripts/test_check_preflight.py index 6cf96eb2..295d1b80 100644 --- a/tests/unit/scripts/test_check_preflight.py +++ b/tests/unit/scripts/test_check_preflight.py @@ -3416,6 +3416,57 @@ def test_a_generic_base_is_followed(self, tmp_path: Path): assert r.status == "FAIL" assert "found via base" in r.details[0] + def test_a_base_chain_is_followed_past_its_first_level(self, tmp_path: Path): + """`multipl_e`'s layout: leaf -> per-protocol base -> shared `feedback`. + + The middle class carries only the protocol and builds no record, so a + walk stopping at the first base finds nothing, and the leaf reports as + `unverified` — a WARN saying "could not read this one", which is exactly + what it says when the declaration underneath is *also* wrong. The four + leaves of that family went unchecked on that shape while the preflight + stayed green. + + `DemoRoot` is reached through `DemoMiddle`'s own module rather than + through an import, which is where a shared base actually lives. + """ + r = self._one( + tmp_path, + "from ._base import DemoMiddle\n" + "@sieval_task(\n name='demo_0shot_gen',\n)\n" + "class DemoTask(DemoMiddle[int]):\n" + " pass\n", + _base="class DemoRoot:\n" + " async def feedback(self, post, ctx):\n" + " return True, build_judgement_record(None, [])\n" + "class DemoMiddle(DemoRoot):\n" + " def prompt_prefix(self, raw):\n" + " return ''\n", + ) + assert r.status == "FAIL" + assert "found via base" in r.details[0] + assert "declare 'procedure'" in r.details[0] + + def test_a_base_outside_the_task_tree_ends_the_walk_quietly(self, tmp_path: Path): + """Every chain ends at something this check cannot open — `core`'s `Task`. + + An unresolvable hop contributes no call sites in either direction, so it + must neither raise nor turn the tier into a finding of its own; the leaf + is then judged on what the chain did yield. + """ + r = self._one( + tmp_path, + "from sieval.core.tasks import Task\n" + "from ._base import DemoBase\n" + "@sieval_task(\n name='demo_0shot_gen',\n" + " reference_kind='procedure',\n)\n" + "class DemoTask(DemoBase, Task[int]):\n" + " pass\n", + _base="class DemoBase:\n" + " async def feedback(self, post, ctx):\n" + " return True, build_judgement_record(None, [])\n", + ) + assert r.status == "PASS" + # --- reach: what the check cannot read, it must not call clean ---------- def test_an_unreachable_task_warns_rather_than_passing_silently( diff --git a/tests/unit/tasks/multipl_e/__init__.py b/tests/unit/tasks/multipl_e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/tasks/multipl_e/conftest.py b/tests/unit/tasks/multipl_e/conftest.py new file mode 100644 index 00000000..0cd8dd32 --- /dev/null +++ b/tests/unit/tasks/multipl_e/conftest.py @@ -0,0 +1,158 @@ +"""Shared doubles for the MultiPL-E task family. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import Request, Response +from sieval.core.models.chat_model import ChatModel +from sieval.core.models.gen_model import GenModel +from sieval.core.tasks import build_judgement_record, build_rollout_judgement +from sieval.datasets.multipl_e_humaneval import MultiPLEHumanEvalDataset +from sieval.datasets.multipl_e_mbpp import MultiPLEMbppDataset +from tests.conftest import HandlerTransport + + +class CapturingGenModel(GenModel): + def __init__(self, text: str = " return 1;"): + self.last_req: Request | None = None + self._text = text + super().__init__(model="mock-gen", api_key="fake") + + def _build_default_transport(self) -> HandlerTransport: + return HandlerTransport(self._stub_arun, "openai_completions") + + async def _stub_arun(self, req: Request) -> Response: + self.last_req = req + return Response(texts=(self._text,) * req.sampling.n) + + +class CapturingChatModel(ChatModel): + def __init__(self, text: str = " return 1;"): + self.last_req: Request | None = None + self._text = text + super().__init__(model="mock-chat", api_key="fake") + + def _build_default_transport(self) -> HandlerTransport: + return HandlerTransport(self._stub_arun, "openai_chat") + + async def _stub_arun(self, req: Request) -> Response: + self.last_req = req + return Response(texts=(self._text,) * req.sampling.n) + + +class EvaluatorDouble: + """Stands in for the code-eval service, recording what was asked of it. + + ``verdicts`` maps a `lang` to the boolean the service should answer with; + anything unlisted passes. ``languages`` is what ``GET /languages`` + advertises, and ``languages=None`` makes the endpoint answer 404 — the + shape of an evaluator that predates the capability probe. + """ + + def __init__( + self, + *, + languages=("cpp", "javascript", "bash", "perl"), + verdicts=None, + msgs=None, + ): + self.bodies: list[dict] = [] + self.deadlines: list[float | None] = [] + self.language_calls: list[str] = [] + self._languages = languages + self._verdicts = verdicts or {} + self._msgs = msgs or {} + + async def get(self, url, *, timeout): + del timeout + self.language_calls.append(url) + if self._languages is None: + return _Response(404, {}) + payload = {"status": True, "msg": "", "data": list(self._languages)} + return _Response(200, payload) + + async def post(self, url, *, json, timeout): + del url + self.bodies.append(json) + self.deadlines.append(timeout) + lang = json["lang"] + ok = self._verdicts.get(lang, True) + return _Response( + 200, + { + "status": ok, + "msg": self._msgs.get(lang, "" if ok else "failed [exit 1]: boom"), + "data": { + "avg_cpu_percent": 0.0, + "peak_cpu_percent": 0.0, + "avg_memory_mb": 0.0, + "peak_memory_mb": 0.0, + "n_cases": 1, + "n_passed": int(ok), + }, + }, + ) + + async def aclose(self) -> None: + return None + + +class UnreachableEvaluator(EvaluatorDouble): + async def get(self, url, *, timeout): + del url, timeout + raise OSError("connection refused") + + +class _Response: + def __init__(self, status_code: int, payload: dict): + self.status_code = status_code + self._payload = payload + + def raise_for_status(self) -> None: + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + def json(self) -> dict: + return self._payload + + +def row( + language: str = "cpp", + *, + name: str = "HumanEval_23_strlen", + prompt: str = "long f(std::string s) {\n", + tests: str = '}\nint main() { assert(f("x") == 1); }', + stop_tokens: tuple[str, ...] = ("\n}",), +) -> dict: + return { + "name": name, + "language": language, + "prompt": prompt, + "doctests": "transform", + "original": "/cluster/path/HumanEval_23_strlen.py", + "prompt_terminology": "reworded", + "tests": tests, + "stop_tokens": list(stop_tokens), + } + + +def dataset_for(rows, *, suite: str = "humaneval"): + cls = MultiPLEHumanEvalDataset if suite == "humaneval" else MultiPLEMbppDataset + return cls(_hf_dict=HFDatasetDict({"test": HFDataset.from_list(rows)})) + + +def judgement(language: str, *verdicts: tuple[bool, str]): + """A JudgementRecord in the shape report() reads.""" + return build_judgement_record( + None, + [ + build_rollout_judgement( + i, correct, extra={"msg": msg, "language": language} + ) + for i, (correct, msg) in enumerate(verdicts) + ], + extra={"name": "n", "language": language}, + ) diff --git a/tests/unit/tasks/multipl_e/test__base.py b/tests/unit/tasks/multipl_e/test__base.py new file mode 100644 index 00000000..9dacc927 --- /dev/null +++ b/tests/unit/tasks/multipl_e/test__base.py @@ -0,0 +1,641 @@ +"""Unit tests for the shared MultiPL-E protocol, grading and reporting. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import dataclasses + +import pytest + +from sieval.core.models import ModelOutput +from sieval.core.tasks import TaskContext, build_prediction_record +from sieval.core.tasks.metrics import interval_declaration_problems +from sieval.tasks.multipl_e._base import ( + DEFAULT_GRADE_WALL, + EVALUATOR_LANG_BY_TAG, + GRADE_WALL_HEADROOM, + _postprocess_chat_completion, + stop_at_stop_token, +) +from sieval.tasks.multipl_e.multipl_e_humaneval_0shot_base_gen import ( + MultiPLEHumanEvalZeroShotBaseGenTask, +) +from sieval.tasks.multipl_e.multipl_e_humaneval_0shot_gen import ( + MultiPLEHumanEvalZeroShotGenTask, +) +from tests.unit.tasks.multipl_e.conftest import ( + CapturingChatModel, + CapturingGenModel, + EvaluatorDouble, + UnreachableEvaluator, + dataset_for, + judgement, + row, +) + + +def _base_task(rows=None, **kwargs): + rows = rows or [row()] + return MultiPLEHumanEvalZeroShotBaseGenTask( + dataset_for(rows), CapturingGenModel(), **kwargs + ) + + +def _chat_task(rows=None, **kwargs): + rows = rows or [row()] + return MultiPLEHumanEvalZeroShotGenTask( + dataset_for(rows), CapturingChatModel(), **kwargs + ) + + +async def _swap(task, evaluator): + """Replace the real http client, as the LiveCodeBench tests do.""" + await task._http_client.aclose() + task._http_client = evaluator + return evaluator + + +# ---- language vocabulary ------------------------------------------------- + + +def test_every_upstream_tag_maps_to_an_evaluator_language(): + from sieval.datasets._multipl_e import HUMANEVAL_LANGUAGES, MBPP_LANGUAGES + + # An unmapped tag would reach setup() as a confusing internal error rather + # than as "the evaluator cannot run this". + assert set(HUMANEVAL_LANGUAGES) <= set(EVALUATOR_LANG_BY_TAG) + assert set(MBPP_LANGUAGES) <= set(EVALUATOR_LANG_BY_TAG) + + +def test_mapping_uses_the_evaluators_existing_spelling(): + # The evaluator names languages in English (`javascript`, not `js`), which + # is the side that has to match for dispatch to work. + assert EVALUATOR_LANG_BY_TAG["js"] == "javascript" + assert EVALUATOR_LANG_BY_TAG["ts"] == "typescript" + assert EVALUATOR_LANG_BY_TAG["sh"] == "bash" + assert EVALUATOR_LANG_BY_TAG["pl"] == "perl" + assert EVALUATOR_LANG_BY_TAG["jl"] == "julia" + assert EVALUATOR_LANG_BY_TAG["ml"] == "ocaml" + assert EVALUATOR_LANG_BY_TAG["rkt"] == "racket" + assert EVALUATOR_LANG_BY_TAG["adb"] == "ada" + + +# ---- stop-token truncation ---------------------------------------------- + + +def test_stop_at_stop_token_takes_the_earliest_match(): + assert stop_at_stop_token("a\n}b\nfunction c", ["\nfunction ", "\n}"]) == "a" + + +def test_stop_at_stop_token_leaves_untouched_text_alone(): + assert stop_at_stop_token(" return 1;", ["\n}"]) == " return 1;" + + +def test_stop_at_stop_token_can_empty_a_completion(): + # Upstream's warning made concrete: scanning text that begins with a stop + # token truncates to nothing, which is why the prompt is never scanned. + assert stop_at_stop_token("\n}", ["\n}"]) == "" + + +# ---- chat postprocessing ------------------------------------------------- + + +def test_chat_postprocess_strips_a_leading_fence(): + reply = "```\nlong f() {\n return 1;\n}\n```" + assert _postprocess_chat_completion(reply, "long f() {\n", []) == ( + "long f() {\n return 1;\n}" + ) + + +def test_chat_postprocess_ignores_a_fence_that_is_not_leading(): + # Upstream only looks at the start of the reply; a mid-reply fence is left + # in place. Kept deliberately -- it is upstream's rule, not an oversight. + reply = "here:\n```\nlong f() { }\n```" + assert "```" in _postprocess_chat_completion(reply, "long f() {\n", []) + + +def test_chat_postprocess_only_scans_past_the_prompt_length(): + # The repeated prefix carries this language's stop token. Scanning from + # zero would truncate the whole program away; upstream's character split is + # what prevents that. + prompt = "long f() {\n" + reply = prompt + " return 1;\n}\nextra" + out = _postprocess_chat_completion(reply, prompt, ["\n}"]) + assert out == prompt + " return 1;" + assert out.startswith(prompt) + + +# ---- program assembly ---------------------------------------------------- + + +@pytest.mark.anyio +async def test_completion_protocol_prepends_the_dataset_prompt(): + task = _base_task() + evaluator = await _swap(task, EvaluatorDouble()) + try: + raw = row() + await task.feedback( + build_prediction_record([" return s.length();"]), + TaskContext(sample_id=0, raw_sample=raw), + ) + program = evaluator.bodies[0]["code"] + # Upstream's assembly, separator included (evaluation/src/main.py). + assert program == raw["prompt"] + " return s.length();" + "\n" + raw["tests"] + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_chat_protocol_discards_the_dataset_prompt(): + # Upstream's blank-prompt rule: the MODEL's copy of the prefix compiles. + # Prepending the dataset prompt would define the function twice, which does + # not compile in most of these languages -- a whole-benchmark zero. + task = _chat_task() + evaluator = await _swap(task, EvaluatorDouble()) + try: + raw = row() + whole_program = raw["prompt"] + " return s.length();" + await task.feedback( + build_prediction_record([whole_program]), + TaskContext(sample_id=0, raw_sample=raw), + ) + program = evaluator.bodies[0]["code"] + assert program == whole_program + "\n" + raw["tests"] + assert program.count(raw["prompt"]) == 1 + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_unextractable_prediction_is_still_run(): + # A blank completion is graded as a real build failure rather than skipped, + # so the verdict comes from the evaluator instead of from a missing record. + task = _base_task() + evaluator = await _swap(task, EvaluatorDouble()) + try: + raw = row() + await task.feedback( + build_prediction_record([None]), + TaskContext(sample_id=0, raw_sample=raw), + ) + assert evaluator.bodies[0]["code"] == raw["prompt"] + "\n" + raw["tests"] + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_feedback_sends_the_language_and_suite_alias(): + task = _base_task([row("sh")]) + evaluator = await _swap(task, EvaluatorDouble()) + try: + await task.feedback( + build_prediction_record(["x"]), + TaskContext(sample_id=0, raw_sample=row("sh")), + ) + body = evaluator.bodies[0] + assert body["lang"] == "bash" + assert body["source"] == "human-eval" + # No `timeout` key: the per-language default on the evaluator is the + # budget, and one number here would charge c++ bash's wall. + assert "timeout" not in body + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_explicit_timeout_is_forwarded_with_headroom(): + task = _base_task(timeout=7.0) + evaluator = await _swap(task, EvaluatorDouble()) + try: + await task.feedback( + build_prediction_record(["x"]), + TaskContext(sample_id=0, raw_sample=row()), + ) + assert evaluator.bodies[0]["timeout"] == 7.0 + # Above the server's own budget, so a slow compile reads as a grade + # rather than as a network timeout. + assert evaluator.deadlines[0] == pytest.approx(7.0 + GRADE_WALL_HEADROOM) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_grade_request_is_never_unbounded(): + # Grading is synchronous on one shared event loop, so an unbounded wait + # stalls the session rather than the sample. Covers the default path, where + # the budget is the evaluator's own per-language one and this side does not + # know it. + task = _base_task() + evaluator = await _swap(task, EvaluatorDouble()) + try: + await task.feedback( + build_prediction_record(["x"]), + TaskContext(sample_id=0, raw_sample=row()), + ) + wall = evaluator.deadlines[0] + assert wall is not None + assert wall == pytest.approx(DEFAULT_GRADE_WALL) + # Has to clear the evaluator's largest row: c++'s 15s run plus its + # separate 60s compile. + assert wall > 75.0 + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_judgement_persists_the_language(): + # So a per-language rate is recomputable from the shard files alone. + task = _base_task([row("pl")]) + await _swap(task, EvaluatorDouble()) + try: + _, record = await task.feedback( + build_prediction_record(["x"]), + TaskContext(sample_id=0, raw_sample=row("pl")), + ) + assert record["extra"]["language"] == "pl" + assert record["rollouts"][0]["extra"]["language"] == "pl" + finally: + await task.shutdown() + + +# ---- infer wiring -------------------------------------------------------- + + +@pytest.mark.anyio +async def test_completion_infer_sends_the_rows_own_stop_tokens(): + # Stop tokens are per language, so they must come off the sample rather + # than from a task constant. + task = _base_task(n=3) + try: + raw = row("js", stop_tokens=("\nfunction ", "\n//")) + ctx = TaskContext(sample_id=0, raw_sample=raw) + pre = await task.preprocess(raw, ctx) + await task.infer(pre, ctx) + req = task.model.last_req + assert req.sampling.stop == ("\nfunction ", "\n//") + assert req.sampling.n == 3 + assert pre["prompt"] == raw["prompt"] + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_chat_infer_sends_no_stop_tokens(): + # The reply is asked to repeat the prefix, whose text contains this + # language's stop tokens -- passing them would cut it off mid-prefix. + task = _chat_task() + try: + raw = row() + ctx = TaskContext(sample_id=0, raw_sample=raw) + pre = await task.preprocess(raw, ctx) + await task.infer(pre, ctx) + assert task.model.last_req.sampling.stop is None + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_chat_prompt_carries_upstreams_instruction(): + task = _chat_task() + try: + raw = row() + pre = await task.preprocess(raw, TaskContext(sample_id=0, raw_sample=raw)) + text = "\n".join(m["content"] for m in pre["prompt"]) + # The "repeat it exactly" instruction is what makes the blank-prompt + # grading path work at all. + assert "do not alter the prefix but repeat it exactly" in text + assert raw["prompt"] in text + assert "cpp" in text + # Upstream's OUTPUT field description, which restates the same + # requirement -- carried, not dropped with the DSPy scaffolding. + assert "complete program including the full prefix" in text + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_chat_prompt_never_sends_two_consecutive_same_role_messages(): + """The instruction goes in `system`, which is where DSPy puts it. + + Asserted on the ROLES, not the joined text -- the text is identical either + way, while a template enforcing strict alternation (Mistral's) rejects two + `user` messages outright and nothing downstream merges them. + """ + task = _chat_task() + try: + raw = row() + pre = await task.preprocess(raw, TaskContext(sample_id=0, raw_sample=raw)) + roles = [m["role"] for m in pre["prompt"]] + assert roles == ["system", "user"] + assert all(a != b for a, b in zip(roles, roles[1:], strict=False)) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_completion_postprocess_truncates_at_the_stop_token(): + task = _base_task() + try: + raw = row() + inferred = ModelOutput( + model=task.model.meta(), texts=[" return 1;\n}\nlong g() {}", " "] + ) + post = await task.postprocess( + inferred, + TaskContext(sample_id=0, raw_sample=raw, infer_result=inferred), + ) + predictions = [r.get("prediction") for r in post["rollouts"]] + assert predictions[0] == " return 1;" + # Whitespace-only survives truncation as itself, not as None: it is + # what the model produced, and the evaluator gives it a real verdict. + assert predictions[1] == " " + finally: + await task.shutdown() + + +# ---- setup probe --------------------------------------------------------- + + +@pytest.mark.anyio +async def test_setup_passes_when_every_language_is_advertised(): + task = _base_task([row("cpp"), row("sh")]) + await _swap(task, EvaluatorDouble(languages=("cpp", "bash"))) + try: + await task.setup() + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_setup_refuses_a_language_the_evaluator_cannot_run(): + # The failure this exists to prevent: without it the run completes, costs a + # full generation budget, and reports pass@1 = 0 with no errors. + task = _base_task([row("cpp"), row("rkt")]) + await _swap(task, EvaluatorDouble(languages=("cpp", "bash"))) + try: + with pytest.raises(ValueError) as excinfo: + await task.setup() + message = str(excinfo.value) + assert "rkt" in message + assert "racket" in message # names the toolchain to deploy + assert "languages" in message # names the way to narrow the run + assert "cpp" in message # names what does work + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_setup_refuses_an_evaluator_without_the_capability_endpoint(): + task = _base_task() + await _swap(task, EvaluatorDouble(languages=None)) + try: + with pytest.raises(RuntimeError, match="languages"): + await task.setup() + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_setup_refuses_an_unreachable_evaluator(): + task = _base_task() + await _swap(task, UnreachableEvaluator()) + try: + with pytest.raises(RuntimeError, match="cannot reach"): + await task.setup() + finally: + await task.shutdown() + + +# ---- report -------------------------------------------------------------- + + +def _finals(*specs, rows): + out = [] + for i, (language, verdicts) in enumerate(specs): + out.append( + dataclasses.replace( + TaskContext(sample_id=i, raw_sample=rows[i]), + feedback_result=judgement(language, *verdicts), + ) + ) + return out + + +@pytest.mark.anyio +async def test_report_publishes_per_language_rates_and_a_macro(): + rows = [row("cpp"), row("sh"), row("js")] + task = _base_task(rows, n=2) + try: + report = await task.report( + _finals( + ("cpp", [(True, ""), (True, "")]), + ("sh", [(True, ""), (False, "failed [exit 1]: x")]), + ("js", [(False, "failed [exit 1]: x")] * 2), + rows=rows, + ), + [], + ) + assert report["pass@1_cpp"] == pytest.approx(100.0) + assert report["pass@1_sh"] == pytest.approx(50.0) + assert report["pass@1_js"] == pytest.approx(0.0) + assert report["n_languages"] == 3 + # Unweighted over languages; the headline pools size-weighted. Equal + # here only because each language has one problem -- asserted anyway so + # the two keys cannot silently become one number. + assert report["pass@1_macro"] == pytest.approx(50.0) + assert report["pass@1"] == pytest.approx(50.0) + assert report["score"] == report["pass@1"] + assert report["score_key"] == "pass@1" + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_report_headline_pools_size_weighted_while_macro_does_not(): + # Two problems in one language, one in another: the two aggregations must + # disagree, which is the whole reason both are published. + rows = [row("cpp", name="a"), row("cpp", name="b"), row("sh")] + task = _base_task(rows) + try: + report = await task.report( + _finals( + ("cpp", [(True, "")]), + ("cpp", [(True, "")]), + ("sh", [(False, "failed [exit 1]: x")]), + rows=rows, + ), + [], + ) + assert report["pass@1"] == pytest.approx(200 / 3) # 2 of 3 problems + assert report["pass@1_macro"] == pytest.approx(50.0) # 100 and 0 + assert report["pass@1"] != report["pass@1_macro"] + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_report_charges_a_pipeline_fail_as_wrong_and_places_its_language(): + # DENOMINATOR_REQUESTED. The failed sample has no judgement to read a + # language from, so this also covers reading it off the live dataset. + rows = [row("cpp"), row("sh")] + task = _base_task(rows) + try: + report = await task.report( + _finals(("cpp", [(True, "")]), rows=rows), + [TaskContext(sample_id=1, raw_sample=rows[1])], + ) + assert report["fails"] == 1 + assert report["pass@1"] == pytest.approx(50.0) + assert report["denominator_policy"] == "requested" + assert report["pass@1_sh"] == pytest.approx(0.0) + assert report["n_problems_sh"] == 1 + assert report["n_languages"] == 2 + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_report_buckets_failures_by_stage(): + rows = [row("cpp"), row("cpp", name="b"), row("cpp", name="c")] + task = _base_task(rows) + try: + report = await task.report( + _finals( + ("cpp", [(False, "failed: timeout")]), + ("cpp", [(False, "failed [build exit 1]: syntax")]), + ("cpp", [(False, "failed [exit 1]: assertion")]), + rows=rows, + ), + [], + ) + assert report["timeouts"] == 1 + assert report["n_build_errors"] == 1 + assert report["n_execution_errors"] == 1 + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_a_build_error_quoting_the_word_timeout_is_not_a_timeout(): + """The bucket is read off the message PREFIX, not by substring. + + A compiler's diagnostic is pasted into the message tail verbatim, so a + program whose build fails on an identifier called `timeout` would otherwise + land in the one bucket that says the model's program ran and was slow. The + verdict is unaffected either way — this is the diagnostic split, and a + diagnostic that lies about which stage failed is worse than none. + """ + rows = [row("cpp"), row("cpp", name="b")] + task = _base_task(rows) + try: + report = await task.report( + _finals( + ("cpp", [(False, "failed [build exit 1]: error: no member 'timeout'")]), + # A build that exceeds its own wall is a BUILD failure: the run + # never started, and build-versus-run is the split being carried. + ("cpp", [(False, "failed: build timeout")]), + rows=rows, + ), + [], + ) + assert report["n_build_errors"] == 2 + assert report["timeouts"] == 0 + assert report["n_execution_errors"] == 0 + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_report_tolerates_a_null_message(): + # A null msg is absent on disk; bucketing must not crash on it. + rows = [row("cpp")] + task = _base_task(rows) + try: + report = await task.report(_finals(("cpp", [(False, None)]), rows=rows), []) + assert report["n_execution_errors"] == 1 + finally: + await task.shutdown() + + +@pytest.mark.parametrize("n", [1, 2]) +@pytest.mark.anyio +async def test_report_interval_declarations_are_consistent(n): + # What the runner checks at report-write time. Guards the `|` merge of + # three fragments: a plain merge can replace `ci95_units` wholesale and + # leave intervals with no unit. + rows = [row("cpp"), row("sh"), row("js"), row("pl")] + task = _base_task(rows, n=n) + try: + report = await task.report( + _finals( + ("cpp", [(True, "")] * n), + ("sh", [(True, "")] * n), + ("js", [(False, "failed [exit 1]: x")] * n), + ("pl", [(False, "failed [exit 1]: x")] * n), + rows=rows, + ), + [], + ) + assert interval_declaration_problems(report) == [] + lo, hi = report["score_ci95"] + assert lo < report["score"] < hi + assert report["n_problems"] == 4 + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_empty_report_declares_without_faking_a_population(): + task = _base_task() + try: + report = await task.report([], []) + assert report["score"] == 0.0 + assert report["score_key"] == "pass@1" + assert report["denominator_policy"] == "requested" + # No zeroed population, which would read as a measured one. + assert "n_problems" not in report + assert "score_ci95" not in report + assert interval_declaration_problems(report) == [] + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_empty_report_writes_every_count_the_full_path_writes(): + """Counts are zeroed on the empty path; only the intervals are absent. + + A key the full report writes and the empty one omits makes the schema depend + on whether any sample survived -- indistinguishable, to a reader diffing two + runs, from a measurement gone missing. + """ + rows = [row("cpp")] + task = _base_task(rows) + try: + full = await task.report( + _finals(("cpp", [(True, "")]), rows=rows), + [], + ) + empty = await task.report([], []) + # Per-LANGUAGE keys are excluded: dimensioned by which languages ran, so + # their absence at zero samples is the schema working, not a hole. + # `pass@1_macro` is not one of those -- it is one key at any population. + per_language = {f"pass@1_{lang}" for lang in ("cpp", "sh", "js", "pl")} + counts = { + key + for key in full + if key.startswith(("n_", "timeouts", "pass@1_", "fails")) + and not key.endswith("_ci95") + and not key.startswith("n_problems_") + and key not in per_language + } + assert counts, "expected the full path to publish counts" + missing = sorted(counts - set(empty)) + assert missing == [], f"empty report omits {missing}" + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_k_greater_than_n_is_refused(): + with pytest.raises(ValueError, match="pass@2"): + _base_task(k=2, n=1) diff --git a/tests/unit/tasks/multipl_e/test_multipl_e_family.py b/tests/unit/tasks/multipl_e/test_multipl_e_family.py new file mode 100644 index 00000000..5b8ec1b5 --- /dev/null +++ b/tests/unit/tasks/multipl_e/test_multipl_e_family.py @@ -0,0 +1,133 @@ +"""One contract over the four MultiPL-E leaves. + +The leaves are near-clones: each pairs a suite with a protocol and adds +nothing else, so what is worth asserting is that the pairing is wired right in +all four rather than re-testing behaviour per leaf. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest + +from sieval.core.tasks.meta import get_task_class, get_task_meta +from sieval.tasks.multipl_e._base import MultiPLEChatTask, MultiPLECompletionTask + +CASES = [ + # (task name, suite, eval_source, model_type, protocol base, dataset) + ( + "multipl_e_humaneval_0shot_base_gen", + "humaneval", + "human-eval", + "gen", + MultiPLECompletionTask, + "multipl_e_humaneval", + ), + ( + "multipl_e_humaneval_0shot_gen", + "humaneval", + "human-eval", + "chat", + MultiPLEChatTask, + "multipl_e_humaneval", + ), + ( + "multipl_e_mbpp_0shot_base_gen", + "mbpp", + "mbpp", + "gen", + MultiPLECompletionTask, + "multipl_e_mbpp", + ), + ( + "multipl_e_mbpp_0shot_gen", + "mbpp", + "mbpp", + "chat", + MultiPLEChatTask, + "multipl_e_mbpp", + ), +] + +NAMES = [case[0] for case in CASES] + + +@pytest.mark.parametrize( + ("name", "suite", "eval_source", "model_type", "protocol", "dataset"), + CASES, + ids=NAMES, +) +def test_leaf_pairs_a_suite_with_a_protocol( + name, suite, eval_source, model_type, protocol, dataset +): + # Resolution by NAME, not by import: a nested task that imports fine but is + # unresolvable shows up as a bare KeyError from `sieval task show` and + # nothing else. + cls = get_task_class(name) + meta = get_task_meta(cls) + + assert issubclass(cls, protocol) + assert cls.suite == suite + assert cls.eval_source == eval_source + assert meta.model_type == model_type + # The FK is resolved from the first generic arg, through a PEP-695 generic + # base -- so a leaf that forgot to parameterize would bind the wrong data. + assert meta.dataset == dataset + + +@pytest.mark.parametrize("name", NAMES) +def test_leaf_declares_a_procedure_reference_and_experimental_status(name): + meta = get_task_meta(get_task_class(name)) + # The ground truth is a test suite, so no reference value is recorded. + assert meta.reference_kind == "procedure" + # No published-score alignment run has been made yet. + assert meta.status == "experimental" + assert meta.n_shot == 0 + + +def _reference(name): + """The leaf's ReferenceImpl, asserted present rather than assumed. + + Every one of these tasks declares one, so a missing block is a regression + worth failing on directly instead of an attribute error further down. + """ + reference = get_task_meta(get_task_class(name)).reference_impl + assert reference is not None + return reference + + +@pytest.mark.parametrize("name", NAMES) +def test_leaf_notes_carry_the_protocol_and_its_traps(name): + reference = _reference(name) + notes = reference.notes + # The upstream repeat protocol differs from this task's default n, so the + # notes have to say how to match it (.claude/rules/tasks.md). + assert "args.n: 20" in notes + assert "temperature" in notes + # Which languages were actually proven, rather than a bare claim of support. + assert "cpp / js / sh / pl" in notes + assert reference.url.startswith("https://github.com/nuprl/MultiPL-E/blob/") + # Commit-pinned, not a branch link. + assert "/blob/main/" not in reference.url + + +@pytest.mark.parametrize("name", NAMES) +def test_chat_leaves_document_the_blank_prompt_rule(name): + meta = get_task_meta(get_task_class(name)) + if meta.model_type != "chat": + return + notes = _reference(name).notes + # The single most consequential detail of the chat protocol: a port that + # misses it scores zero across the benchmark. + assert "blank prompt" in notes + assert "define the function twice" in notes + + +MBPP_NAMES = ["multipl_e_mbpp_0shot_base_gen", "multipl_e_mbpp_0shot_gen"] + + +@pytest.mark.parametrize("name", MBPP_NAMES) +def test_mbpp_leaves_record_the_suites_own_quirks(name): + notes = _reference(name).notes + assert "no `mbpp-dart` config" in notes + # Upstream's translator rewrote "python" inside the docstrings too. + assert "cppthon" in notes diff --git a/vendor/code-evaluator/README.md b/vendor/code-evaluator/README.md index dedc86fb..d548b5b6 100644 --- a/vendor/code-evaluator/README.md +++ b/vendor/code-evaluator/README.md @@ -8,6 +8,7 @@ code either by running it directly or by checking it against test cases. ## Supported datasets - HumanEval (multi-language: python / javascript / typescript) +- MultiPL-E (cpp / bash / perl, plus javascript via the HumanEval path) - LiveCodeBench (python only) - SciCode (python only) @@ -60,6 +61,7 @@ docker build -f docker/Dockerfile.python -t code-evaluator-py . docker build -f docker/Dockerfile.javascript -t code-evaluator-js . docker build -f docker/Dockerfile.typescript -t code-evaluator-ts . docker build -f docker/Dockerfile.scicode -t code-evaluator-scicode . # Python 3.11 +docker build -f docker/Dockerfile.multipl-e -t code-evaluator-multipl-e . # g++ / bash / perl / node ``` ## Running the service @@ -74,6 +76,22 @@ fastapi run app/server.py --port 11451 GET /health -> `{"status": true, "msg": "healthy"}` +### Language capabilities + +GET /languages -> `{"status": true, "msg": "", "data": ["bash", "cpp", ...]}` + +Which `lang` values this deployment accepts. A caller runs this **before** +spending inference: without it, an unsupported language is only discoverable +one sample at a time, by which point everything has been generated and the +report reads as a model that scored zero rather than as an evaluator that +cannot run the language. + +It answers for **this image**, not the source table: a table-driven language +whose toolchain is not on `PATH` is withheld, and `POST /evaluations` refuses it +the same way (naming the missing command) instead of failing at spawn. A `PATH` +lookup, not an invocation — ~0.1 ms for the whole table. It proves the entry +point exists, not that the toolchain works. + ### Evaluation endpoint POST /evaluations @@ -82,7 +100,8 @@ Fields: - `uuid` - `source`: `"human-eval"` | `"mbpp"` | `"livecodebench"` | `"scicode"` -- `lang`: `python` | `javascript` | `typescript` +- `lang`: `python` | `javascript` | `typescript` | `cpp` | `bash` | `perl` — + ask `GET /languages` rather than hard-coding this list - `code`: the code, as a string - `test`: LiveCodeBench-specific test description (`fn_name` / `inputs` / `outputs`) - `timeout`: float (optional, seconds; defaults below) — a wall for the **whole suite** @@ -170,15 +189,37 @@ When a request omits `timeout`, these defaults apply: - python / js: 3s - typescript: 5s +- bash / perl: 3s +- cpp: 15s for the program, plus a separate fixed 60s for the compile — two + budgets, as upstream MultiPL-E gives each step its own. Charging a slow + compile against the program's wall would fail a correct submission on a cold + cache; charging it nothing would let a pathological one hang. - livecodebench: 6s + 2s * number of cases, or `(timeout_per_case + 1) * n + 5` when `timeout_per_case` was sent — upstream's own backstop shape +For the table-driven languages (cpp / bash / perl) the wall is enforced against +the whole **process group**: the program is spawned in its own session and a +timeout `SIGKILL`s the group, so a submission that forks cannot outlive its +budget or leave the child running after the response. This is what upstream +MultiPL-E's `safe_subprocess` does. Note that these walls are the service's own +and do not match upstream's flat 15s-per-step in either direction — see +`VENDORED.md`. + ### Memory limit When a request omits `memory_limit`, the default is 1024 MB. - Python: limits the process address space via `resource.setrlimit`. - Node.js (JS/TS): limits the V8 heap via `--max-old-space-size`. +- Table-driven languages (cpp / bash / perl): limits the address space with a + `ulimit -v` shell prologue that `exec`s the program, so its exit code, + signals and streams pass through untouched. Applied to the program only — the + compiler is trusted toolchain cost, and a cap the kernel refuses is left + unset rather than failing the submission. Verified binding: a 1 GiB + allocation is refused at `memory_limit=256` and succeeds uncapped. + An address-space cap does **not** suit a managed runtime — a JVM or Go + program reserves far more virtual space than it commits — so a future `java` / + `scala` / `go` row needs a cgroup RSS cap or the runtime's own heap flag. ## Layout diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index 1cd1d056..ad85ea9e 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -72,3 +72,95 @@ - `README.md` — translated from Chinese to English, so the vendored docs match the rest of the repo. Content is otherwise unchanged apart from the case-count section above. +- `app/exec_lang.py` (new), `app/server.py`, `docker/Dockerfile.multipl-e` + (new), `README.md` — **table-driven languages, for MultiPL-E.** Adds `cpp`, + `bash` and `perl` to the direct-run path as rows in one declarative table + (`ext` + build argv + run argv + budgets) behind a single executor, instead of + a fourth, fifth and sixth copy of the write-file/spawn/wait/decode/classify + sequence that `exec_py_code` / `exec_js` / `exec_ts` each hand-roll. Those + three modules are **untouched**, so no existing task's grading moves; the + `CODE_EXECUTOR_MAP` merely moved to module scope (it was rebuilt per request) + and gains the table's rows. + + Commands follow upstream MultiPL-E's `evaluation/src/eval_.py` + (`g++ -std=c++17`, `bash path`, `perl path`). Two of its conventions are + carried deliberately: + + - **perl fails on `ERROR` in the output even at exit 0** (`eval_pl.py`). + Checked only on an otherwise-passing run, so it can turn a pass into a + failure and never the reverse. Not reachable through the shipped test + templates — 0 of 161 `humaneval-pl` rows mention `ERROR`, and all 161 signal + failure with `exit 1` — so it fires only on model stdout. Kept because it is + upstream's rule, not because the data exercises it. + - **A build gets its own wall** (60s) separate from the program's, as each + `safe_subprocess.run` call does upstream. c++ takes upstream's own 15s for + the program. + + The **wall clocks themselves diverge from upstream**, in both directions, and + the tasks' `reference_impl.notes` enumerate it. Upstream gives every step a + flat 15s — each `eval_.py` call is a bare `safe_subprocess.run` taking + its default, the build included. Here an interpreted row keeps the service's + own 3s run default rather than tripling every other benchmark's wall to match + one, and a build gets 60s. So a submission needing between 3s and 15s to run + fails here and passes upstream, and a compile upstream abandons at 15s + completes here. The run wall has measured headroom rather than an assumed one: + a pure-bash prime sieve to n=20000 finishes in 0.6s. + + A **timed-out program is killed by process group** — spawned with + `start_new_session=True` and reaped with `os.killpg(..., SIGKILL)` — which is + what upstream's own `safe_subprocess` does, and for the reason its comment + gives ("Without this line, test_fork_once fails"). Killing only the direct + child leaves the wall unenforced whenever the submission forks: measured + before the fix, `bash` running a program that spawns a 30s sleeper returned + after **30.1s against a 3s budget**, and the sleeper outlived the kill. After + it, 3.1s and no survivor. The reap is itself bounded (`KILL_GRACE_SECONDS`), + so a process wedged in uninterruptible sleep cannot re-introduce the unbounded + wait. `exec_js` / `exec_ts` have the same shape and are **not** fixed here — + they are untouched by this patch on purpose, and their fix belongs in its own + change. + + Upstream's four-way failure taxonomy (`SyntaxError` / `AssertionError` / + `ReferenceError` / `Exception`) is **not** reproduced: the response carries one + boolean and all four buckets are the same boolean. What is preserved is the + part a caller cannot reconstruct — build failure vs run failure — spelled in + `msg`, reusing the vocabulary the hand-rolled modules already established + (`failed: timeout`, `failed [exit N]: ...`). + + Memory is capped with a `ulimit -v` prologue that `exec`s the program, not + `preexec_fn`: neither sibling module uses one, and it runs between fork and + exec inside a threaded server, where a lock another thread holds at fork is + held forever in the child. Verified binding rather than assumed — a 1 GiB + allocation is refused at `memory_limit=256` and succeeds uncapped, and a + trivial program still passes under the cap. + + **Also fixes a latent bug** on exactly this path: for an unsupported `lang` + the direct-run branch never bound `timeout`, which the `logger.info` below it + reads unconditionally, so the request died with `UnboundLocalError` (a 500) + instead of returning the branch's own `not supported language: `. That is + the path every MultiPL-E language whose toolchain is not deployed takes — + 20 of 24 today — so it would have been hit immediately. Reproduced before and + after against the real endpoint. +- `app/server.py`, `app/exec_lang.py`, `README.md` — **`GET /languages`**, + advertising the `lang` values a deployment accepts. A caller probes it + *before* spending inference: without it, an unsupported language is + discoverable only per sample, by which point every sample is generated and the + report reads as a model that scored zero rather than an evaluator that cannot + run the language. + + It answers for the **image**, not the source table: `toolchain_present` + resolves each row's entry command on `PATH` (the *build* command where there + is one — a compiled row's `run` names the compiler's output, not anything + installed), and an absent one is withheld. `POST /evaluations` applies the + same test, so a missing toolchain reads as ``not supported language: cpp (row + exists but `g++` is not on PATH in this image)`` rather than a per-sample + `FileNotFoundError` that looks like the model's fault. Without it the guard + only moved the silent-zeros failure from "language not in the table" to "row + present, compiler absent" — reachable by deploying any of the other + Dockerfiles with this server. + + An existence check, not an invocation: ~0.1 ms for the whole table, so nothing + is compiled to answer a probe. A broken install still fails per sample. + + The two entries above are **not yet upstream** — land them in + `scitix/code-evaluator` and re-vendor. Their tests belong there rather than + under `tests/`, which mirrors `sieval/`. diff --git a/vendor/code-evaluator/app/exec_lang.py b/vendor/code-evaluator/app/exec_lang.py new file mode 100644 index 00000000..e53c8c69 --- /dev/null +++ b/vendor/code-evaluator/app/exec_lang.py @@ -0,0 +1,308 @@ +"""Table-driven execution for compiled and interpreted languages. + +One executor, one row per language. ``exec_py_code`` / ``exec_js`` / ``exec_ts`` +each hand-roll the same write-file, spawn, wait, decode, classify sequence; a +fourth, fifth and sixth copy of it would be six places to fix the next bug in +the resource-monitor handshake. What actually differs between languages is data +-- a file extension, a build command, a run command, a budget -- so it is held +as data here and the sequence is written once. + +Adding a language is a :data:`LANGUAGES` row plus its toolchain in the image. +No new module, and nothing in ``server.py`` to edit. + +The classification is deliberately coarser than upstream MultiPL-E's, which +sorts a failure into ``SyntaxError`` / ``AssertionError`` / ``ReferenceError`` / +``Exception``. This service's response carries one boolean, and every one of +those buckets is the same boolean: the distinction is diagnostic, not scoring. +What survives is the part a caller cannot reconstruct -- whether the failure was +the BUILD or the RUN -- because it is spelled in ``msg``. + +Messages reuse the vocabulary the hand-rolled modules already established +(``failed: timeout``, ``failed [exit N]: ...``), so a caller that greps for one +does not have to learn a second spelling for the same thing. +""" + +import asyncio +import os +import shutil +import signal +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +from .resource_monitor import ResourceStats, monitor_process_resources + +#: How long to wait for a killed process group to be reaped before giving up on +#: it. A SIGKILLed group is gone in microseconds; this exists so that a process +#: wedged in uninterruptible sleep (a stuck NFS read) cannot re-introduce the +#: unbounded wait the kill is here to remove. +KILL_GRACE_SECONDS = 5.0 + + +@dataclass(frozen=True) +class LanguageSpec: + """How to build and run one language's program. + + ``build`` and ``run`` are argv templates: ``{path}`` is the source file and + ``{stem}`` the same path without its extension, which is where a compiler is + told to put its output. A language with no build step leaves ``build`` None. + + Two budgets rather than one, because upstream gives each step its own + ``safe_subprocess.run`` call with its own wall: charging a slow compile + against the program's budget would fail a correct program on a cold cache, + and charging the compile nothing would let a pathological one hang. Only + ``timeout`` (the run) is caller-supplied; ``build_timeout`` is fixed here, + since it is a property of the toolchain rather than of the submission. + + ``fail_on_output`` carries a language whose test harness reports failure by + PRINTING rather than by exiting non-zero -- upstream's perl rule. Empty for + every other row, and it is checked only on an otherwise-passing run, so it + can turn a pass into a failure and never the reverse. + """ + + ext: str + run: tuple[str, ...] + build: tuple[str, ...] | None = None + default_timeout: float = 3.0 + build_timeout: float = 60.0 + fail_on_output: tuple[str, ...] = field(default=()) + + +# COMMANDS follow upstream MultiPL-E's `evaluation/src/eval_.py`, so a +# program that passes here is one that passes there -- modulo the budgets below, +# which are this service's and not upstream's: +# cpp -- eval_cpp.py: `g++ ... -std=c++17`, build failure is its own status +# bash -- eval_sh.py: `bash path`, exit code only +# perl -- eval_pl.py: `perl path`, exit code AND "ERROR" in the output +# +# Upstream gives every one of those a flat 15s, because each is a bare +# `safe_subprocess.run` call taking its default -- the build included. Two rows +# here differ, in opposite directions, and both are deliberate: +# * interpreted rows keep the 3s default the service already uses for js and +# python, rather than tripling every other benchmark's wall to match one. +# Measured headroom rather than a guess: a pure-bash prime sieve to n=20000 +# finishes in 0.6s, two hundred times under the wall. A submission that +# would need between 3s and 15s fails here and passes upstream. +# * a build gets its own 60s, since charging a cold-cache compile against the +# program's budget fails a correct program for the toolchain's slowness. The +# other direction: a compile upstream would abandon at 15s completes here. +# Both are enumerated in the tasks' `reference_impl.notes` as divergences. +LANGUAGES: dict[str, LanguageSpec] = { + "cpp": LanguageSpec( + ext=".cpp", + build=("g++", "{path}", "-o", "{stem}", "-std=c++17"), + run=("{stem}",), + default_timeout=15.0, + ), + "bash": LanguageSpec(ext=".sh", run=("bash", "{path}")), + "perl": LanguageSpec( + ext=".pl", + run=("perl", "{path}"), + # eval_pl.py fails a run whose output mentions ERROR even at exit 0. + fail_on_output=("ERROR",), + ), +} + + +def _argv(template: tuple[str, ...], path: Path) -> list[str]: + stem = str(path.with_suffix("")) + return [part.format(path=str(path), stem=stem) for part in template] + + +def toolchain_entry(spec: LanguageSpec) -> str: + """The command *spec* needs on PATH for anything to run at all. + + The BUILD command where there is one: it runs first, and a compiled row's + ``run`` names the compiler's OUTPUT (``{stem}``), not anything installed. + """ + return spec.build[0] if spec.build else spec.run[0] + + +def toolchain_present(spec: LanguageSpec) -> bool: + """Whether *spec*'s entry command resolves on PATH. + + Existence, not health -- a PATH lookup, so a capability endpoint can call it + every time. It is what lets that endpoint mean "this deployment can run the + row" rather than "the source tree has a row for it"; advertising a row the + image cannot run costs a full generation budget on programs that fail at + spawn. + """ + return shutil.which(toolchain_entry(spec)) is not None + + +def _capped(argv: list[str], memory_limit: int | None) -> list[str]: + """Wrap *argv* so the program runs under an address-space cap, or return it. + + A shell prologue rather than ``preexec_fn``, which neither sibling module + uses and which is unsafe here for the reason it is unsafe anywhere: it runs + between fork and exec inside a THREADED server, where any lock another + thread holds at fork is held forever in the child. js caps via + ``NODE_OPTIONS`` and python via ``setrlimit`` inside its own forked child, + so nothing in this service pays that risk today and this row should not be + the first. + + ``exec`` replaces the shell, so the program's own exit code, signals and + streams pass through untouched -- the shell is gone by the time it runs. A + ``ulimit`` the kernel refuses is left to the shell's own reporting rather + than aborting: a cap that cannot be set is the platform's answer, and + failing the submission over it would score a model on the host's limits. + + ``-v`` caps the ADDRESS SPACE, which suits every row today (a c++ binary, + bash, perl) but not a managed runtime: a JVM or Go program reserves far more + virtual space than it commits, so ``java`` / ``scala`` / ``go`` would fail + to start under any useful cap rather than be limited by it. Those need + another mechanism (a cgroup RSS cap, or the runtime's own heap flag as + ``exec_js`` uses ``--max-old-space-size``) -- so memory is a third thing a + new language needs decided, beside its toolchain and its test harness's + dependencies. + """ + if not memory_limit: + return argv + kb = int(memory_limit) * 1024 + return ["/bin/sh", "-c", f'ulimit -v {kb} 2>/dev/null; exec "$0" "$@"', *argv] + + +async def _kill_group(proc: asyncio.subprocess.Process) -> None: + """SIGKILL a timed-out process AND everything it forked. + + ``proc.kill()`` alone signals the direct child. Anything that child forked + inherited this process's stdout and stderr, so an orphan holding those pipes + keeps them open, and the ``proc.wait()`` that has to follow the kill blocks + until *it* exits -- not until the budget is spent. The timeout then bounds + nothing: the verdict is still ``failed: timeout``, but the call returns + whenever the orphan decides to, which for a program that backgrounded a + sleeper or started a server is far past the wall or never. Measured before + this existed: `bash` running a program that forks a 30s sleeper returned + after 30.1s against a 3s budget, and the sleeper outlived the kill. + + Upstream MultiPL-E's ``safe_subprocess`` does exactly this, and says why in + a comment: "Kills the process group. Without this line, test_fork_once + fails." The group is the session made at spawn time, so it holds the + submission and its descendants and nothing belonging to this server. + + ``ProcessLookupError`` is the good case -- the program exited between the + timeout firing and the signal landing -- and is not worth reporting. + """ + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + try: + await asyncio.wait_for(proc.wait(), timeout=KILL_GRACE_SECONDS) + except asyncio.TimeoutError: + # Unreapable within the grace period. Returning leaves the child to the + # event loop's watcher, which is strictly better than the caller + # inheriting an unbounded wait -- the thing this function exists to stop. + pass + + +async def _spawn( + argv: list[str], *, cwd: str, timeout: float, memory_limit: int | None, monitor: bool +) -> tuple[int | None, str, ResourceStats, bool]: + """Run *argv* to completion. Returns (exit code, stderr, stats, timed_out). + + An exit code of None means the process was killed on the timeout, so it + never reported one -- the same "unknown, not zero" convention the response's + ``n_passed`` uses. + """ + stats = ResourceStats() + proc = await asyncio.create_subprocess_exec( + *_capped(argv, memory_limit), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + env=os.environ.copy(), + # Its own session, so the timeout has a process GROUP to kill rather + # than one pid -- see `_kill_group`. Set at spawn because a group cannot + # be established after the fact, and `_capped`'s shell prologue `exec`s + # the program, so the leader is the submission itself either way. + start_new_session=True, + ) + + if monitor and proc.pid is not None: + stats, stop_event = await monitor_process_resources(proc.pid) + else: + stop_event = asyncio.Event() + stop_event.set() + + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + # Both streams: upstream's perl rule reads stdout as well as stderr, and + # a build error is on stderr. Joined so one scan covers both. + output = f"{stdout.decode(errors='replace')}\n{stderr.decode(errors='replace')}" + return proc.returncode, output.strip(), stats, False + except asyncio.TimeoutError: + await _kill_group(proc) + return None, "", stats, True + finally: + stop_event.set() + if monitor: + # Give the monitor time to finish -- but only if one was started. + # The build step has none, so this cost 0.1s per compiled grade for + # nothing. + await asyncio.sleep(0.1) + + +async def execute_code( + code: str, + timeout: float, + memory_limit: int | None = None, + *, + spec: LanguageSpec, +) -> tuple[bool, str, ResourceStats]: + """Build (if the language needs it) and run *code*, one all-or-nothing case. + + Matches the ``(ok, msg, stats)`` contract of the per-language modules, so + ``server.py`` dispatches to it the same way. *spec* is keyword-only and has + no default: a caller reaching this without naming a language is a bug, not a + request to guess one. + + The stats returned are the RUN's, never the build's -- the build is + toolchain cost, and reporting a compiler's memory as the submission's would + make a c++ row incomparable with an interpreted one. + """ + stats = ResourceStats() + try: + with tempfile.TemporaryDirectory() as workdir: + # Named for the language, in its own directory: a compiler writes + # its output next to the source, and the directory is what bounds + # the mess a submission can make of the filesystem. + path = Path(workdir) / f"program{spec.ext}" + path.write_text(code, encoding="utf-8") + + if spec.build is not None: + exit_code, output, _, timed_out = await _spawn( + _argv(spec.build, path), + cwd=workdir, + timeout=spec.build_timeout, + # The toolchain is trusted and can be memory-hungry; the cap + # exists for the submission, which is the next step. + memory_limit=None, + monitor=False, + ) + if timed_out: + return False, "failed: build timeout", stats + if exit_code != 0: + # Upstream calls this SyntaxError. Spelled as a build + # failure because that is what it is for an interpreted + # language too, where a syntax error surfaces at run time. + return False, f"failed [build exit {exit_code}]: {output}", stats + + exit_code, output, stats, timed_out = await _spawn( + _argv(spec.run, path), + cwd=workdir, + timeout=timeout, + memory_limit=memory_limit, + monitor=True, + ) + if timed_out: + return False, "failed: timeout", stats + if exit_code != 0: + return False, f"failed [exit {exit_code}]: {output}", stats + for marker in spec.fail_on_output: + if marker in output: + return False, f"failed [output contains {marker!r}]: {output}", stats + return True, output, stats + except Exception as e: + return False, f"failed: [{type(e).__name__}] {e}", stats diff --git a/vendor/code-evaluator/app/server.py b/vendor/code-evaluator/app/server.py index 11a75137..702ba493 100644 --- a/vendor/code-evaluator/app/server.py +++ b/vendor/code-evaluator/app/server.py @@ -1,6 +1,7 @@ import atexit import os import sys +from functools import partial from typing import Any, Generic, TypeVar from fastapi import FastAPI @@ -8,6 +9,8 @@ from pydantic import BaseModel from .exec_js import execute_code as exec_js +from .exec_lang import LANGUAGES, toolchain_entry, toolchain_present +from .exec_lang import execute_code as exec_lang from .exec_py_code import execute_code as exec_py_code from .exec_py_test import execute_test as exec_py_test from .exec_ts import execute_code as exec_ts @@ -79,11 +82,54 @@ class ResourceMetrics(BaseModel): n_passed: int | None = None +# Direct-run executors, by `lang`. The three hand-rolled modules keep their own +# rows; every table-driven language contributes one automatically, so a new +# toolchain is a row in `exec_lang.LANGUAGES` and nothing here. +CODE_EXECUTOR_MAP: dict[str, tuple[Any, float]] = { + "javascript": (exec_js, 3.0), + "python": (exec_py_code, 3.0), + "typescript": (exec_ts, 5.0), +} | { + name: (partial(exec_lang, spec=spec), spec.default_timeout) + for name, spec in LANGUAGES.items() +} + + @app.get("/health") async def check_health() -> BasicResponse[None]: return BasicResponse(status=True, msg="healthy") +@app.get("/languages") +async def list_languages() -> BasicResponse[list[str]]: + """Which `lang` values this deployment can actually run, sorted. + + A capability probe, so a caller can refuse a run BEFORE spending inference + on it. Without one, an unsupported language is only discoverable per sample, + at which point every sample has already been generated and the report reads + as a model that scored zero rather than as an evaluator that cannot run the + language. Advertising the set is what makes that distinguishable. + + It answers for THIS DEPLOYMENT: a table row whose toolchain is missing from + the image is withheld, not advertised. Otherwise the guard only moves the + silent-zeros failure from "language not in the table" to "row present, + compiler absent" -- the same run of zeros it exists to prevent. + + An existence check, not an invocation: a PATH lookup (~0.1 ms for the whole + table), so nothing is compiled to answer a probe. It proves the entry point + exists, not that the toolchain works. + """ + return BasicResponse( + status=True, + msg="", + data=sorted( + lang + for lang in CODE_EXECUTOR_MAP + if lang not in LANGUAGES or toolchain_present(LANGUAGES[lang]) + ), + ) + + class LiveCodeBenchTest(BaseModel): inputs: list[str] outputs: list[str] @@ -113,19 +159,31 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: # so a clean run == pass, same as human-eval. logger.debug(f"code to exec:\n{sample.code}") - CODE_EXECUTOR_MAP = { - "javascript": (exec_js, 3.0), - "python": (exec_py_code, 3.0), - "typescript": (exec_ts, 5.0), - } - if sample.lang in CODE_EXECUTOR_MAP: + spec = LANGUAGES.get(sample.lang) + if sample.lang in CODE_EXECUTOR_MAP and ( + spec is None or toolchain_present(spec) + ): fn, default_timeout = CODE_EXECUTOR_MAP[sample.lang] timeout = sample.timeout if sample.timeout is not None else default_timeout ok, msg, stats = await fn( code=sample.code, timeout=timeout, memory_limit=sample.memory_limit ) else: + # `timeout` is bound on this path too: the log line below reads it + # unconditionally, so leaving it unset raised UnboundLocalError and + # answered a 500 instead of this branch's own message -- and this is + # exactly the path a language whose toolchain is not deployed takes. + timeout = sample.timeout + # A row whose toolchain is absent lands here rather than at a + # `FileNotFoundError` from the spawn, so this and `GET /languages` + # answer alike. Naming the missing command separates a deployment + # gap from a per-sample failure that reads as the model's. ok, msg = False, f"not supported language: {sample.lang}" + if spec is not None: + msg = ( + f"{msg} (row exists but `{toolchain_entry(spec)}` is not on " + f"PATH in this image)" + ) stats = None logger.info( diff --git a/vendor/code-evaluator/docker/Dockerfile.multipl-e b/vendor/code-evaluator/docker/Dockerfile.multipl-e new file mode 100644 index 00000000..60f5bf89 --- /dev/null +++ b/vendor/code-evaluator/docker/Dockerfile.multipl-e @@ -0,0 +1,48 @@ +FROM python:3.10-slim + +# MultiPL-E toolchains for the table-driven languages in app/exec_lang.py. +# One image rather than one per language: MultiPL-E is graded per language but +# run as one sweep, and a caller that has to route each `lang` to its own +# endpoint gains nothing the table does not already give it. +# +# g++ -- humaneval-cpp / mbpp-cpp (build step; `bits/stdc++.h`, which +# every cpp prompt includes, is GCC's) +# bash -- humaneval-sh / mbpp-sh (in the base image; named here so a +# base change that drops it is visible) +# perl -- humaneval-pl / mbpp-pl +# nodejs -- humaneval-js / mbpp-js (served by app/exec_js.py; its tests +# `require('assert')`, a Node built-in) +# +# A test harness's own dependencies matter as much as the toolchain, and they +# fail differently: a missing compiler is one error, a missing test library is +# 100% of that language failing to compile while every other language is fine. +# `libtest-deep-perl` is the live case -- EVERY MultiPL-E perl test opens with +# `use Test::Deep`, so without it the language scores a clean 0. Upstream's own +# `evaluation/Dockerfile` installs it for the same reason. Adding a language +# means reading its test template for this, not just installing its compiler. +# +# A language absent here is still advertised by GET /languages, which answers +# for the source table rather than for the image; it fails at spawn. Add a +# toolchain and its `exec_lang.LANGUAGES` row together, or the endpoint promises +# what the image cannot run. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + g++ \ + bash \ + perl \ + libtest-deep-perl \ + curl && \ + curl -sL https://deb.nodesource.com/setup_22.x | bash - && \ + apt-get install -y --no-install-recommends nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# python evaluator +WORKDIR /opt/code-evaluator + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ ./app/ + +CMD ["fastapi", "run", "app/server.py", "--port", "11451"]