feat(tasks): add Ag-LiveCodeBench-X — multi-language LiveCodeBench

sieval/datasets/__init__.pyi

@@ -9,6 +9,10 @@ from .advanced_if import (
     AdvancedIFDataset,
     AdvancedIFDatasetSample,
 )
+from .ag_livecodebench_x import (
+    AgLiveCodeBenchXDataset,
+    AgLiveCodeBenchXDatasetSample,
+)
 from .agieval import (
     AGIEvalDataset,
     AGIEvalDatasetSample,
@@ -259,6 +263,8 @@ __all__ = [
     "ARCEasyDatasetSample",
     "AdvancedIFDataset",
     "AdvancedIFDatasetSample",
+    "AgLiveCodeBenchXDataset",
+    "AgLiveCodeBenchXDatasetSample",
     "Apex2025Dataset",
     "Apex2025DatasetSample",
     "ApexShortlist2025Dataset",

sieval/datasets/ag_livecodebench_x.py

@@ -0,0 +1,87 @@
+"""Ag-LiveCodeBench-X — the multi-language LiveCodeBench subset from Agnostics.
+
+Upstream (NUPRL, `Agnostics <https://arxiv.org/abs/2508.04865>`_) took
+LiveCodeBench v5's 880 problems, kept the **499 that specify and check their
+answer purely through stdin/stdout**, and dropped the 381 that ship Python
+starter code. What is left is language-agnostic: the same problem statement and
+the same test cases score a solution written in any language, which is why the
+set can be pointed at Lua, R, Julia, OCaml or Fortran without re-authoring
+anything.
+
+Three columns and no more — there is no ``starter_code``, no ``metadata`` and no
+``public_test_cases`` split here, so every case is a private one and the
+call-based (``fn_name``) grading path LiveCodeBench also carries cannot arise.
+``private_test_cases`` keeps LiveCodeBench's own encoding (base64 -> zlib ->
+pickle -> JSON); decoded, the 499 problems are ~8 GB of test data, which is why
+it travels compressed.
+
+Upstream publishes **no license**: the Hub card carries no license field and the
+scripts repo has no ``LICENSE`` file, so ``license`` is ``None`` rather than a
+guess. See the task's ``reference_impl.notes`` for what that costs in port
+fidelity.
+
+AI-Generated Code - Claude Opus 5 (Anthropic)
+"""
+
+from typing import TypedDict, override
+
+from datasets import DatasetDict as HFDatasetDict
+from datasets import load_dataset
+
+from sieval.core.datasets import (
+    Category,
+    Dataset,
+    Level1Category,
+    sieval_dataset,
+)
+from sieval.core.utils.hf import ensure_dataset_dict
+
+AG_LIVECODEBENCH_X_REVISION = "15bef6bf966893163ea9d22f25b7c58fec06a949"
+
+
+class AgLiveCodeBenchXDatasetSample(TypedDict):
+    question_id: str
+    question_content: str
+    private_test_cases: str
+
+
+@sieval_dataset(
+    name="ag_livecodebench_x",
+    display_name="Ag-LiveCodeBench-X",
+    description=(
+        "LiveCodeBench v5's 499 stdin/stdout problems, prepared by Agnostics "
+        "for any language."
+    ),
+    source=f"hf:nuprl/Ag-LiveCodeBench-X@{AG_LIVECODEBENCH_X_REVISION}",
+    categories=(
+        Category(Level1Category.CODE, "CodeGeneration"),
+        Category(Level1Category.CODE, "MultiLanguageSupport"),
+    ),
+    # No `python` tag, unlike `livecodebench_code_generation`: the target
+    # language is the task's `language` argument, not a property of the data.
+    tags=("english", "code-exec"),
+    # Upstream declares none (no Hub license field, no LICENSE in the scripts
+    # repo). `None` is the accurate record; do not infer one from LiveCodeBench,
+    # whose own label ('cc') covers a different artifact.
+    license=None,
+)
+class AgLiveCodeBenchXDataset(Dataset[AgLiveCodeBenchXDatasetSample]):
+    @override
+    def load(self, name_or_path: str, **kwargs) -> HFDatasetDict:
+        # One call for both a Hub id and a staged directory: the release is plain
+        # parquet under `data/`, with no loader script and no config to select,
+        # so `load_dataset` resolves either form on its own. No `trust_remote_code`
+        # for the same reason -- unlike `livecodebench_code_generation`, whose Hub
+        # copy ships a builder script.
+        dataset = ensure_dataset_dict(load_dataset(name_or_path, **kwargs))
+
+        # A window that filters everything away would otherwise be reported as a
+        # score of 0.0 rather than as a mistake.
+        if any(len(split) == 0 for split in dataset.values()):
+            raise ValueError(
+                "Ag-LiveCodeBench-X dataset is empty after loading "
+                f"(name_or_path={name_or_path!r}); the release has a single "
+                "'test' split of 499 problems."
+            )
+
+        return dataset

sieval/meta/index.json

@@ -45,6 +45,31 @@
       "license": "CC-BY-NC-4.0",
       "checksums": {}
     },
+    {
+      "name": "ag_livecodebench_x",
+      "display_name": "Ag-LiveCodeBench-X",
+      "description": "LiveCodeBench v5's 499 stdin/stdout problems, prepared by Agnostics for any language.",
+      "source": [
+        "hf:nuprl/Ag-LiveCodeBench-X@15bef6bf966893163ea9d22f25b7c58fec06a949"
+      ],
+      "categories": [
+        {
+          "level1": "Code",
+          "level2": "CodeGeneration"
+        },
+        {
+          "level1": "Code",
+          "level2": "MultiLanguageSupport"
+        }
+      ],
+      "tags": [
+        "english",
+        "code-exec"
+      ],
+      "deps_group": null,
+      "license": null,
+      "checksums": {}
+    },
     {
       "name": "agieval",
       "display_name": "AGIEval",
@@ -1445,6 +1470,28 @@
       "status": "experimental",
       "reference_kind": "procedure"
     },
+    {
+      "name": "ag_livecodebench_x_0shot_gen",
+      "display_name": "Ag-LiveCodeBench-X (0-shot)",
+      "description": "Competitive programming in an arbitrary language, graded by the Agnostics verifier.",
+      "dataset": "ag_livecodebench_x",
+      "eval_mode": "gen",
+      "n_shot": 0,
+      "tags": [
+        "english",
+        "code-exec",
+        "multi-language"
+      ],
+      "deps_group": null,
+      "model_type": "chat",
+      "reference_impl": {
+        "source": "nuprl/Ag-LiveCodeBench-X",
+        "url": "https://github.com/nuprl/Ag-LiveCodeBench-X/blob/b7b273ef5710db814d6eff8bca3c4f432b60ad4b/livecodebench_x.py",
+        "notes": "Prompt is the frozen dspy==3.0.0b2 ChatAdapter rendering of upstream's ChainOfThought(SolveProblem) -- captured, not read off the signature, because the 8-space instruction indent and the trailing-space asymmetry in the field list are only observable at runtime; pinned by hash in the task's tests. Response parsing and code extraction are REIMPLEMENTED, not vendored: upstream ships no license (no Hub license field, no LICENSE file). Divergences: (1) upstream's ChatAdapter retries a failed parse through JSONAdapter with a differently shaped prompt, which this port does not -- so a reply missing either marker scores 0 here and the gap is exactly n_unextracted, making this a lower bound; (2) the Agnostics verifier container is run by sieval's code-evaluator rather than from the harness process, and pinned by DIGEST rather than by upstream's mutable tag -- the digest that scored each rollout comes back on the verdict and is surfaced as report()'s verifier_image; (3) upstream's pre-run 'Say this is a test!' smoke call is not reproduced. Verifier tags are upstream's own, which are FILE EXTENSIONS, not language names -- ghcr.io/nuprl/agnostics ships lua, r, python, jl, java, cpp, ml, f90, so Julia/OCaml/Fortran resolve to jl/ml/f90 and a lowercased language name would name no image. Upstream defaults: n=1 completion, temperature 0.6 / top_p 0.95 / max_tokens 5000, container timeout supplied per run (README uses 15s, and sends that one number as BOTH the container's timeout_s and the outer process wall). ALIGNMENT: Table 1's Qwen 3 8B x Lua cell, published 11, measured 10.42 (52/499) against the real ghcr.io/nuprl/agnostics lua verifier at its pinned digest -- 0.58 pp, or 0.30 combined binomial sigma, at upstream's README settings for Qwen 3 (temperature 0.2, top_p 0.95, max_tokens 2048, language 'Lua /nothink'). Caveats that do not move it out of noise: the paper served qwen3_8b_awq (AWQ 4-bit) where this used a provider reporting unknown quantization; 13 of 499 rollouts never generated (provider rate-limits) and count as failures, biasing the number down to 10.42 from 10.70; and divergence (1) cost exactly one row. The other four languages and every other model in the table are unmeasured."
+      },
+      "status": "stable",
+      "reference_kind": "procedure"
+    },
     {
       "name": "agieval_0shot_gen",
       "display_name": "AGIEval (0-shot, generative)",

sieval/tasks/__init__.pyi

@@ -7,6 +7,9 @@ from .aa_lcr_0shot_gen import (
 from .advanced_if_0shot_gen import (
     AdvancedIFZeroShotGenTask,
 )
+from .ag_livecodebench_x_0shot_gen import (
+    AgLiveCodeBenchXZeroShotGenTask,
+)
 from .agieval_0shot_gen import (
     AGIEvalZeroShotGenTask,
 )
@@ -247,6 +250,7 @@ __all__ = [
     "ARCEasyFewShotClpTask",
     "ARCEasyFewShotPplTask",
     "AdvancedIFZeroShotGenTask",
+    "AgLiveCodeBenchXZeroShotGenTask",
     "Apex2025ZeroShotGenTask",
     "ApexShortlist2025ZeroShotGenTask",
     "BRUMO2025ZeroShotGenTask",