From 34a0ea24e1b5702a0c6121bb786bcba298d5fdb3 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sun, 23 Aug 2026 00:44:37 +0800 Subject: [PATCH 1/5] =?UTF-8?q?feat(tasks):=20add=20LiveOIBench=20?= =?UTF-8?q?=E2=80=94=20olympiad=20C++=20problems=20with=20subtask=20scorin?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 403 informatics-olympiad problems from 72 contests across 14 olympiads (2023-2025), graded in C++ with IOI-style subtask partial credit and ranked against the contests' real human contestants. This is the first benchmark here whose grading needs a compiler. Every other code benchmark posts to the vendored code-evaluator, which ran Python, JS and TS only; LiveOIBench is C++-first (the paper measures C++, and reports Python ~9 points lower on a USACO subset), so the port has two halves: * `vendor/code-evaluator` gains `source="liveoibench"` — g++ compile, then one child per test case under RLIMIT_CPU / RLIMIT_AS at the problem's own limits with upstream's explicit 20% buffer, plus its CPU poller and its output comparison. Compilation is bounded where upstream's is not, every test always runs (subtask scoring needs the whole verdict vector), and the response carries `case_verdicts` + `case_names` so the caller maps verdicts onto subtasks by the names the server used rather than re-deriving an ordering. * `sieval/community/liveoibench/` ports the prompt, extraction (vendored byte-identical), subtask scoring and human ranking from the Apache-2.0 upstream at 7759e3b8. Scope is the 380 `batch` problems. The 23 interactive ones need an interactor process the evaluator does not run, so the dataset filters them out rather than scoring them zero — which means these numbers are NOT comparable to the paper's 403-problem table. The 5 script-judged problems run `evaluate.sh` shipped as data and are excluded with them. No checker path exists, and that is fidelity rather than a shortcut: the published dataset materializes no `checkers/` directory, so upstream's own judge compares outputs directly on this data and the 36 min-score subtasks (of 3700) collapse onto the all-or-nothing rule. The 33.5 GB test corpus ships as three single-row-group parquets, so no cheap per-problem read exists; `scripts/materialize_liveoibench_tests.py` unpacks it once into upstream's own layout, and the loader refuses to run until it has, naming the command. The evaluator then reads test directories off disk — inlining would move the whole corpus over HTTP once per rollout. Upstream samples n=8 and reports the best candidate; the task defaults to n=1 and reproduces best-of-n at whatever n it is given. No published number is claimed: no run has been aligned, and the task ships with no alignment card. Verified against g++ 14.2: unit tests for prompt, scoring, ranking, dataset and task (64 new, 5856 total green), the evaluator's verdict vectors for correct / partial / TLE / MLE / compile-error / float-tolerance / grader-linked submissions, and a full end-to-end run of dataset -> prompt -> real g++ over HTTP -> subtask scores -> human percentile. Co-Authored-By: Claude Opus 5 (1M context) --- examples/README.md | 1 + examples/liveoibench.yaml | 81 +++ scripts/materialize_liveoibench_tests.py | 128 +++++ sieval/community/liveoibench/__init__.py | 0 .../community/liveoibench/code_extractor.py | 208 ++++++++ sieval/community/liveoibench/payloads.py | 43 ++ sieval/community/liveoibench/prompts.py | 74 +++ sieval/community/liveoibench/rankings.py | 165 ++++++ sieval/community/liveoibench/scoring.py | 74 +++ sieval/datasets/__init__.pyi | 6 + sieval/datasets/liveoibench.py | 263 ++++++++++ sieval/meta/index.json | 46 ++ sieval/tasks/__init__.pyi | 4 + sieval/tasks/liveoibench_0shot_gen.py | 491 ++++++++++++++++++ tests/unit/community/liveoibench/__init__.py | 0 .../community/liveoibench/test_prompts.py | 80 +++ .../community/liveoibench/test_rankings.py | 146 ++++++ .../community/liveoibench/test_scoring.py | 108 ++++ tests/unit/datasets/test_liveoibench.py | 216 ++++++++ .../unit/tasks/test_liveoibench_0shot_gen.py | 443 ++++++++++++++++ vendor/code-evaluator/VENDORED.md | 47 ++ vendor/code-evaluator/app/exec_cpp.py | 406 +++++++++++++++ vendor/code-evaluator/app/server.py | 120 +++++ vendor/code-evaluator/docker/Dockerfile.cpp | 20 + 24 files changed, 3170 insertions(+) create mode 100644 examples/liveoibench.yaml create mode 100644 scripts/materialize_liveoibench_tests.py create mode 100644 sieval/community/liveoibench/__init__.py create mode 100644 sieval/community/liveoibench/code_extractor.py create mode 100644 sieval/community/liveoibench/payloads.py create mode 100644 sieval/community/liveoibench/prompts.py create mode 100644 sieval/community/liveoibench/rankings.py create mode 100644 sieval/community/liveoibench/scoring.py create mode 100644 sieval/datasets/liveoibench.py create mode 100644 sieval/tasks/liveoibench_0shot_gen.py create mode 100644 tests/unit/community/liveoibench/__init__.py create mode 100644 tests/unit/community/liveoibench/test_prompts.py create mode 100644 tests/unit/community/liveoibench/test_rankings.py create mode 100644 tests/unit/community/liveoibench/test_scoring.py create mode 100644 tests/unit/datasets/test_liveoibench.py create mode 100644 tests/unit/tasks/test_liveoibench_0shot_gen.py create mode 100644 vendor/code-evaluator/app/exec_cpp.py create mode 100644 vendor/code-evaluator/docker/Dockerfile.cpp diff --git a/examples/README.md b/examples/README.md index efeb5b4b..988dbc1c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,7 @@ matches what you're trying to do, copy it, edit the marked fields, and run | [infer-recipe-override.yaml](infer-recipe-override.yaml) | Pin a specific inference recipe or override engine args | | [ruler-qwen3-8b-nonthinking.yaml](ruler-qwen3-8b-nonthinking.yaml) | RULER long-context sweep — 13 subtasks × 6 lengths (4k–128k), YaRN for 64k/128k | | [ruler-qwen3-8b-thinking.yaml](ruler-qwen3-8b-thinking.yaml) | Same RULER sweep with Qwen3 thinking enabled (adds `think_budget` + reasoning parser) | +| [liveoibench.yaml](liveoibench.yaml) | LiveOIBench — olympiad C++ problems scored by subtask; needs a C++-capable evaluator and a one-time 33.5 GB test unpack | ## Hardware-indexed (reference configs) diff --git a/examples/liveoibench.yaml b/examples/liveoibench.yaml new file mode 100644 index 00000000..60728c18 --- /dev/null +++ b/examples/liveoibench.yaml @@ -0,0 +1,81 @@ +# ------------------------------------------------------------------------------ +# LiveOIBench — informatics-olympiad problems, graded in C++ by subtask +# ------------------------------------------------------------------------------ +# 403 problems from 72 contests across 14 olympiads (2023–2025). Scoring is +# IOI-style: a subtask pays its points only when every test in it passes, and the +# report ranks the model against the contest's real human contestants. +# +# Three-step flow, and step 2 is the one people miss: +# 1. sieval dataset download liveoibench # ~33.5 GB — the official tests +# 2. python scripts/materialize_liveoibench_tests.py +# 3. sieval eval liveoibench.yaml +# +# Step 2 unpacks the test corpus into per-problem directories. The three parquet +# files each hold a single row group, so there is no cheap per-problem read out +# of them — the loader refuses to run until they are unpacked, and says so. +# Already have upstream's `$LIVEOIBENCH_ROOT/data` tree? Point at it instead: +# args: {tests_root: /path/to/LIVEOIBENCH_ROOT/data} +# +# THIS NEEDS AN EVALUATOR WITH THE C++ SOURCE. Grading compiles the submission +# with g++ and runs it under the problem's own time and memory limits, which the +# base evaluator image cannot do — build it from +# `vendor/code-evaluator/docker/Dockerfile.cpp` and point SIEVAL_CODE_EVAL_API at +# it. The evaluator also reads the test directory off disk, so it needs the data +# volume mounted; if it cannot see it, set `inline_tests: true` below and accept +# ~140 MB per rollout on the wire. +# +# Scope: this runs the 380 `batch` problems. The 23 interactive ones need an +# interactor process the evaluator does not run yet, so the dataset filters them +# out rather than scoring them zero — which means these numbers are NOT +# comparable to the paper's 403-problem table. Say which you ran next to a score. +# +# Edit these fields to match your setup: +# models.local-model.infer.checkpoint — your model's on-disk path +# models.local-model.infer_meta.image — swap to your own image if you need +# custom deps, or drop the block to run +# without a container +# ------------------------------------------------------------------------------ + +result_dir: ./outputs/liveoibench + +models: + local-model: + args: + concurrency_limit: 32 + temperature: 1.0 + infer: + backend: sglang + checkpoint: /path/to/your/model # EDIT ME + infer_meta: + gpu: H100-80G + image: lmsysorg/sglang:latest + +datasets: + liveoibench: + class: LiveOIBenchDataset + # `sieval dataset download liveoibench` stages the problems repo here; the + # tests and contestant repos land beside it and are found from this path. + path: "${SIEVAL_DATA_DIR}/LiveOIBench/LiveOIBench" + args: + # Narrow the sweep while you are setting up — one competition is enough to + # prove the toolchain end to end: + # competition: IOI + # year: "2025" + task_type: batch + +tasks: + liveoibench_0shot_gen: + class: LiveOIBenchZeroShotGenTask + dataset: liveoibench + model: local-model + args: + # Upstream samples 8 candidates per problem and reports the BEST one, which + # is what the published percentiles are. At n=1 the score is a different + # (lower) quantity — reproduce upstream's protocol or say you did not. + n: 8 + # Set true only if the evaluator cannot read the materialized test tree. + inline_tests: false + infer_args: + # Olympiad solutions are long, and a truncated program compiles to nothing. + # Watch `n_truncated` in report.json and raise this if it is non-zero. + max_tokens: 32768 diff --git a/scripts/materialize_liveoibench_tests.py b/scripts/materialize_liveoibench_tests.py new file mode 100644 index 00000000..af9b9714 --- /dev/null +++ b/scripts/materialize_liveoibench_tests.py @@ -0,0 +1,128 @@ +"""Unpack the LiveOIBench test corpus into per-problem test directories. + +The published test data is three parquet files (2023/2024/2025, 33.5 GB total), +each a **single row group** whose ``tests`` column holds every problem's cases as +one JSON blob per row. That layout has no cheap random access: reading one +problem's tests means decompressing the whole column chunk. So the corpus is +unpacked once, into the directory layout upstream's ``process_dataset.py`` +produces: + + {out}/{competition}/{year}/{round}/{task}/tests/{case}.in + {out}/{competition}/{year}/{round}/{task}/tests/{case}.out + +Rows stream one at a time, so peak memory is one problem's tests (~83 MB on +average) rather than the file. Expect >30 GB written and a long run; already +materialized problems are skipped unless ``--overwrite`` is given. + +Equivalent to upstream's ``process_dataset.py --skip-metadata``, minus the +problem-metadata tree that sieval reads from the parquet directly. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +import pyarrow.parquet as pq + +ROOT = Path(__file__).resolve().parents[1] +# Import sieval from THIS checkout rather than through the editable install, +# which may point at a different worktree. +sys.path.insert(0, str(ROOT)) + +from sieval.datasets.liveoibench import ( # noqa: E402 + TEST_YEARS, + problem_tests_dir, + year_parquet_name, +) + + +def materialize_year( + parquet_path: Path, tests_root: Path, overwrite: bool, verbose: bool = True +) -> tuple[int, int]: + """Unpack one year file. Returns (problems written, problems skipped).""" + written = skipped = 0 + parquet_file = pq.ParquetFile(parquet_path) + # batch_size=1 keeps one problem's payload in memory at a time; the column + # projection drops `subtasks`, which the dataset loader reads separately. + for batch in parquet_file.iter_batches( + batch_size=1, columns=["problem_id", "tests"] + ): + for problem_id, payload in zip( + batch.column("problem_id").to_pylist(), + batch.column("tests").to_pylist(), + strict=True, + ): + out_dir = Path(problem_tests_dir(str(tests_root), problem_id)) + if out_dir.is_dir() and not overwrite: + skipped += 1 + continue + tests = json.loads(payload or "{}") + if not isinstance(tests, dict): + raise ValueError(f"Malformed tests payload for {problem_id!r}") + out_dir.mkdir(parents=True, exist_ok=True) + for name, case in tests.items(): + case = case or {} + (out_dir / f"{name}.in").write_text( + case.get("input", ""), encoding="utf-8" + ) + (out_dir / f"{name}.out").write_text( + case.get("output", ""), encoding="utf-8" + ) + written += 1 + if verbose: + print(f" {problem_id}: {len(tests)} cases") + return written, skipped + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--tests-repo", + default=os.path.join( + os.environ.get("SIEVAL_DATA_DIR", "data"), + "LiveOIBench", + "LiveOIBench_tests", + ), + help="Directory holding the staged liveoibench_testcases_v1_*.parquet files.", + ) + parser.add_argument( + "--out", + default=None, + help="Where to write test directories (default: /materialized, " + "which is what the dataset loader looks for).", + ) + parser.add_argument("--years", nargs="*", default=list(TEST_YEARS)) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args() + + tests_repo = Path(args.tests_repo) + tests_root = Path(args.out) if args.out else tests_repo / "materialized" + + total_written = total_skipped = 0 + for year in args.years: + parquet_path = tests_repo / year_parquet_name(year) + if not parquet_path.exists(): + print( + f"missing {parquet_path}; " + "run `sieval dataset download liveoibench` first" + ) + return 1 + print(f"unpacking {parquet_path.name} -> {tests_root}") + written, skipped = materialize_year( + parquet_path, tests_root, args.overwrite, verbose=not args.quiet + ) + print(f" {year}: {written} written, {skipped} already present") + total_written += written + total_skipped += skipped + + print(f"done: {total_written} problems written, {total_skipped} skipped") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sieval/community/liveoibench/__init__.py b/sieval/community/liveoibench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/community/liveoibench/code_extractor.py b/sieval/community/liveoibench/code_extractor.py new file mode 100644 index 00000000..a39517ab --- /dev/null +++ b/sieval/community/liveoibench/code_extractor.py @@ -0,0 +1,208 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/code_extractor.py +""" +Utility for extracting code from LLM responses. + +This module provides a centralized way to extract and save code from +various LLM response formats. +""" + +import re +import os +import json +from typing import Optional, Tuple + + +class CodeExtractor: + """Utility class for extracting code from LLM responses.""" + + @staticmethod + def extract_code(content: str, task_name: str) -> Tuple[Optional[str], str]: + """ + Extract code from LLM response content using prioritized patterns. + + Args: + content: The response content from the LLM + task_name: Name of the task (used for pattern matching) + + Returns: + Tuple of (extracted_code, status) where status is one of: + 'success', 'empty', 'not_found' + """ + if not content: + return None, 'not_found' + + # Prioritized patterns for code extraction + patterns = [ + # 1) ```{task}.cpp or ```{task}.c ... ``` + rf"```(?:{re.escape(task_name)}\.(?:cpp|c))\s*([\s\S]*?)```", + # 2) ```cpp or ```c ... ``` + r"```(?:cpp|c)\s*([\s\S]*?)```", + # 3) Generic ``` ... ``` + r"```([\s\S]*?)```", + ] + + code = None + for pattern in patterns: + matches = re.findall(pattern, content, re.DOTALL) + if matches: + # Get the longest code block + code = max(matches, key=lambda x: len(x.split('\n'))).strip() + break + + if code is None: + return None, 'not_found' + + if not code: + return "", 'empty' + + # Post-process the code + code = CodeExtractor._post_process_code(code) + return code, 'success' + + @staticmethod + def _post_process_code(code: str) -> str: + """ + Post-process extracted code to fix common issues. + + Args: + code: Raw extracted code + + Returns: + Cleaned code + """ + lines = code.split('\n') + + # Check if the code is too short + if len(lines) < 5: + # This might be an issue but we'll keep it + pass + + # Fix first line if it contains just "cpp" or ends with .cpp + if lines: + first_line = lines[0].strip() + if first_line.lower() == "cpp" or first_line.endswith(".cpp"): + lines[0] = "// " + first_line + + return '\n'.join(lines) + + @staticmethod + def save_code( + code: Optional[str], + prediction_path: str, + task: str, + model: str, + seed: int + ): + """ + Save extracted code to a file. + + Args: + code: Extracted code (can be None or empty) + prediction_path: Path to save predictions + task: Task name + model: Model name + seed: Random seed number + """ + # Create codes directory if it doesn't exist + os.makedirs(os.path.join(prediction_path, "codes"), exist_ok=True) + + # Save code (even if empty) + code_path = os.path.join(prediction_path, "codes", f"{task}_{model}_{seed}.cpp") + with open(code_path, "w") as f: + f.write(code if code else "") + + @staticmethod + def save_raw_response( + response: any, + prediction_path: str, + task: str, + model: str, + seed: int + ): + """ + Save raw API response to a JSON file. + + Args: + response: API response object + prediction_path: Path to save predictions + task: Task name + model: Model name + seed: Random seed number + """ + if response is None: + return + + # Skip empty responses + if hasattr(response, "choices") and hasattr(response, "usage"): + if response.usage.completion_tokens == 0: + return + + # Create raw directory if it doesn't exist + os.makedirs(os.path.join(prediction_path, "raw"), exist_ok=True) + + # Save raw response + raw_path = os.path.join(prediction_path, "raw", f"{task}_{model}_{seed}.json") + + # Handle different response formats + try: + if hasattr(response, "model_dump_json"): + with open(raw_path, "w") as f: + f.write(response.model_dump_json()) + elif hasattr(response, "to_dict"): + with open(raw_path, "w") as f: + json.dump(response.to_dict(), f) + else: + with open(raw_path, "w") as f: + json.dump(response, f) + except Exception as e: + print(f"Warning: Could not save raw response for {task} seed {seed}: {e}") + + @staticmethod + def extract_from_response(response: any) -> Optional[str]: + """ + Extract content from an API response object. + + Args: + response: API response object + + Returns: + Response content as string, or None if not available + """ + if response is None: + return None + + if not hasattr(response, "choices") or len(response.choices) == 0: + return None + + if hasattr(response.choices[0], "message") and hasattr(response.choices[0].message, "content"): + return response.choices[0].message.content + + return None + + +class ExtractionStats: + """Track code extraction statistics.""" + + def __init__(self): + self.success = 0 + self.failed = 0 + self.empty = 0 + + def record(self, status: str): + """Record an extraction result.""" + if status == 'success': + self.success += 1 + elif status == 'empty': + self.empty += 1 + else: + self.failed += 1 + + def print_summary(self): + """Print extraction statistics summary.""" + print("\nCode Extraction Statistics:") + print(f" Successful extractions: {self.success}") + print(f" Failed extractions: {self.failed}") + print(f" Empty code blocks: {self.empty}") + + def __str__(self): + return f"Success: {self.success}, Failed: {self.failed}, Empty: {self.empty}" diff --git a/sieval/community/liveoibench/payloads.py b/sieval/community/liveoibench/payloads.py new file mode 100644 index 00000000..1d706b08 --- /dev/null +++ b/sieval/community/liveoibench/payloads.py @@ -0,0 +1,43 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/process_dataset.py +# (_normalize_code_mapping, and the subtasks parse in write_problem_assets) +"""Parsing the JSON-in-a-column payloads the LiveOIBench parquets carry. + +Divergence from upstream: upstream decodes these while writing files to disk and +raises ``DatasetError`` on a malformed subtasks blob; here the same decoding +happens at load time. A malformed *code* bundle still reads as "this problem +ships none", which is upstream's behaviour and the common case — 362 of 403 +problems have no grader bundle at all. +""" + +import json +import math + + +def load_subtasks(raw_subtasks: str | None) -> dict[str, dict]: + """Parse the ``subtasks`` column into upstream's ``{id: subtask}`` mapping.""" + parsed = json.loads(raw_subtasks or "{}") + if not isinstance(parsed, dict): + raise ValueError(f"Malformed subtasks payload: {type(parsed).__name__}") + return parsed + + +def load_code_bundle(raw_value: object) -> dict[str, str]: + """Parse a ``grader_code`` / ``starter_code`` cell into filename → contents.""" + if raw_value is None: + return {} + if isinstance(raw_value, float) and math.isnan(raw_value): + return {} + if isinstance(raw_value, bytes): + raw_value = raw_value.decode("utf-8") + if not isinstance(raw_value, str): + return {} + stripped = raw_value.strip() + if not stripped: + return {} + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return {} + if not isinstance(parsed, dict): + return {} + return {str(name): str(contents) for name, contents in parsed.items()} diff --git a/sieval/community/liveoibench/prompts.py b/sieval/community/liveoibench/prompts.py new file mode 100644 index 00000000..010cb4e2 --- /dev/null +++ b/sieval/community/liveoibench/prompts.py @@ -0,0 +1,74 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/problem.py +# (Problem.get_prompt) and .../src/process_dataset.py (ParquetReconstructor._write_prompt), +# which build the same string from the same fields — one from a reconstructed +# problem directory, the other from the parquet row. +"""Prompt construction for LiveOIBench. + +Divergence from upstream: upstream reads ``statements/statement.md`` and the +``attachments/`` directory off disk, after ``process_dataset.py`` has +materialized them. Here the same values arrive as the parquet fields they were +written from — ``problem_statement`` and the ``starter_code`` bundle (a JSON +map of filename to contents) — so the file-existence tests become key lookups. +The assembled string is unchanged, including the ``time_limit`` / +``memory_limit`` floats rendering as ``1.5`` / ``2048.0``. + +Upstream's grader block requires all three of ``grader.cpp`` (or ``stub.cpp``), +``{task}.h`` and ``{task}.cpp``; 17 of the 403 published problems satisfy that, +and the rest take the plain branch. +""" + +from typing import Mapping + + +def build_prompt( + task: str, + statement: str, + time_limit: float | None, + memory_limit: float | None, + starter_codes: Mapping[str, str] | None = None, +) -> str: + """Return the user prompt for one problem. + + *task* is upstream's task token — the ``problem_id`` remainder after + competition/year/round, which the dataset carries as ``task_name``. It names + the fence the model is asked to use, so it is also what + ``CodeExtractor.extract_code`` matches on. + """ + starter_codes = starter_codes or {} + + prompt = ( + f"Given a competition problem below, write a solution in C++ that solves all the subtasks. " + f"Make sure to wrap your code in '```{task}.cpp' and '```' Markdown delimiters.\n\n" + ) + if statement: + prompt += "[BEGIN PROBLEM]\n" + prompt += statement + prompt += "[END PROBLEM]\n" + + time_text = time_limit if time_limit is not None else "unknown" + memory_text = memory_limit if memory_limit is not None else "unknown" + prompt += f"Time limit: {time_text} seconds\n" + prompt += f"Memory limit: {memory_text} MB\n" + + grader_code = starter_codes.get("grader.cpp") + if grader_code is None: + grader_code = starter_codes.get("stub.cpp") + header_code = starter_codes.get(f"{task}.h") + solution_code = starter_codes.get(f"{task}.cpp") + + if grader_code is not None and header_code is not None and solution_code is not None: + prompt += ( + "We are going to grade your solution using the following grader.cpp file and " + f"{task}.h file. You can write your solution by modifying the {task}.cpp and wrap " + f"your code in '```{task}.cpp' and '```' Markdown delimiters.\n\n" + ) + prompt += "```grader.cpp\n" + grader_code + "```\n\n" + prompt += f"```{task}.h\n" + header_code + "```\n\n" + prompt += f"```{task}.cpp\n" + solution_code + "```\n\n" + else: + prompt += ( + f"Generate a solution in C++ that solves the task. Make sure to wrap your code in " + f"'```{task}.cpp' and '```' Markdown delimiters.\n\n" + ) + + return prompt diff --git a/sieval/community/liveoibench/rankings.py b/sieval/community/liveoibench/rankings.py new file mode 100644 index 00000000..c79f4810 --- /dev/null +++ b/sieval/community/liveoibench/rankings.py @@ -0,0 +1,165 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/generate_rankings.py +# (normalize_name, NON_TASK_COLUMNS, identify_task_columns, calculate_percentile, +# and the per-contest branch of compute_human_metrics) +"""Ranking a model against the contest's human contestants. + +Divergence from upstream: + +* Upstream operates on a pandas DataFrame built from ``contestants_ranking``; + here the rows arrive as the list of dicts that column actually holds, so + ``df.columns`` becomes the union of row keys and the vectorized column sum + becomes a per-row sum. The numbers are the same. +* Upstream prefers a ``Recalculated_Total`` column when one exists and, on that + branch only, re-derives the Canadian Computing Olympiad percentile from + ``Rank``. No published contest carries that column, so neither branch is + ported; ``score_contest`` implements the column-matching branch that the + released data takes. +* Codeforces Elo is not ported — it is a separate upstream fit against + contestant CF ratings. + +Human totals are re-summed over **only** the tasks the model was scored on, so a +contest whose problems were partly filtered out (interactive ones are) still +compares like with like. Medal cutoffs are the contest's published ones and are +not rescaled, which is upstream's behaviour. +""" + +import re +from typing import Any, Mapping, Sequence + +NON_TASK_COLUMNS = { + "rank", + "contestant", + "country", + "total", + "recalculated_total", + "medal", + "cf_rating", + "day1", + "day2", + "day 1", + "day 2", + "score rel.", + "division", + "team", + "nationality", +} + +_FALLBACK_TOTAL_COLUMNS = ("Total", "total", "Total Score", "score", "Score") + + +def normalize_name(value: str) -> str: + """Normalize task column names for consistent matching.""" + return re.sub(r"[^a-z0-9]", "", value.lower()) + + +def identify_task_columns(columns: Sequence[str], contest_id: str = "") -> dict[str, str]: + """Return a mapping from normalized task name to original column name.""" + mapping: dict[str, str] = {} + normalized_exclusions = {normalize_name(col) for col in NON_TASK_COLUMNS} + contest_lower = (contest_id or "").lower() + + # BOI scores by day, so `day1` / `day2` are tasks there rather than headers. + if "boi-" in contest_lower: + for day_col in ("day1", "day2"): + normalized_exclusions.discard(day_col) + + for column in columns: + norm = normalize_name(column) + if norm and norm not in normalized_exclusions: + mapping[norm] = column + return mapping + + +def calculate_percentile(model_score: float, human_scores: Sequence[float]) -> float | None: + """Percent of contestants the model strictly outscores.""" + if not human_scores: + return None + better = sum(1 for score in human_scores if model_score > score) + return (better / len(human_scores)) * 100 + + +def medal_from_cutoffs( + total: float, + gold_cutoff: float | None, + silver_cutoff: float | None, + bronze_cutoff: float | None, +) -> str | None: + """Upstream's cutoff ladder: ``None`` only when the contest publishes none.""" + if gold_cutoff is not None and total >= gold_cutoff: + return "Gold" + if silver_cutoff is not None and total >= silver_cutoff: + return "Silver" + if bronze_cutoff is not None and total >= bronze_cutoff: + return "Bronze" + if any(cutoff is not None for cutoff in (gold_cutoff, silver_cutoff, bronze_cutoff)): + return "None" + return None + + +def _to_numeric(value: Any) -> float: + """``pd.to_numeric(errors="coerce").fillna(0.0)`` for a single cell.""" + if value is None or isinstance(value, bool): + return 0.0 + try: + number = float(value) + except (TypeError, ValueError): + return 0.0 + return 0.0 if number != number else number # NaN -> 0.0 + + +def score_contest( + rankings: Sequence[Mapping[str, Any]], + model_scores: Mapping[str, float], + contest_id: str = "", + gold_cutoff: float | None = None, + silver_cutoff: float | None = None, + bronze_cutoff: float | None = None, +) -> dict: + """Rank one model's contest total against that contest's contestants. + + *model_scores* maps a problem's ``task_name`` to the model's score on it, for + the problems of this contest that were actually evaluated. + """ + columns: list[str] = [] + for row in rankings: + for key in row: + if key not in columns: + columns.append(key) + task_column_map = identify_task_columns(columns, contest_id) + + matched_columns: list[str] = [] + matched_scores: list[float] = [] + for task_name, score in model_scores.items(): + column = task_column_map.get(normalize_name(str(task_name))) + if column: + matched_columns.append(column) + matched_scores.append(score) + + if matched_columns: + model_total = float(sum(matched_scores)) + else: + # No per-task columns: fall back to the contest total, against which the + # model's own total is every evaluated problem summed. + fallback = next((col for col in _FALLBACK_TOTAL_COLUMNS if col in columns), None) + if fallback is None: + return { + "human_percentile": None, + "medal": None, + "model_total": float(sum(model_scores.values())), + "matched_columns": [], + "n_contestants": len(rankings), + } + matched_columns = [fallback] + model_total = float(sum(model_scores.values())) + + human_totals = [ + sum(_to_numeric(row.get(column)) for column in matched_columns) for row in rankings + ] + + return { + "human_percentile": calculate_percentile(model_total, human_totals), + "medal": medal_from_cutoffs(model_total, gold_cutoff, silver_cutoff, bronze_cutoff), + "model_total": model_total, + "matched_columns": matched_columns, + "n_contestants": len(rankings), + } diff --git a/sieval/community/liveoibench/scoring.py b/sieval/community/liveoibench/scoring.py new file mode 100644 index 00000000..675df43b --- /dev/null +++ b/sieval/community/liveoibench/scoring.py @@ -0,0 +1,74 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/base_judge.py +# (BaseJudge.interprete_task_result) +"""Subtask scoring for LiveOIBench. + +Divergence from upstream: none in the arithmetic. Upstream is a method on the +judge that has just executed the tests; here the per-test verdicts arrive from +the evaluator service, so the same function takes them as an argument. The +misspelled name is upstream's and is kept so the two can be diffed. + +The ``min-score`` branch is reachable only with a custom checker, which supplies +a fractional per-test ``score``. No checker ships in the published dataset (see +``process_dataset.py``, which materializes no ``checkers/`` directory), so on +that data every test scores 1.0 or 0.0 and the branch collapses onto the +all-or-nothing rule below it. It is ported anyway, because dropping it would +make the port wrong for any release that does ship checkers. +""" + +from typing import Any, Mapping, Sequence + + +def interprete_task_result( + results: Sequence[Mapping[str, Any]], + subtasks: Mapping[str, Mapping[str, Any]], +) -> dict: + """Score one submission from its per-test verdicts. + + *results* is one mapping per executed test with ``test_case`` (the test's + name, no extension), ``correct`` (bool) and optionally ``score`` (float, + checker-supplied). *subtasks* is the problem's ``subtasks`` payload: an id + to ``{"score": int, "testcases": [name, ...]}`` mapping, optionally with + ``"grading": "min-score"``. + + Returns ``{"subtasks": {id: {...}}, "score": int, "tests_passed": float, + "ace": bool}``. + """ + score: dict[str, Any] = {"subtasks": {}} + results_dict = {result["test_case"]: result for result in results} + test_cases_passed = [result["test_case"] for result in results if result["correct"]] + + for i, subtask in subtasks.items(): + subtask_score = 0 + subtask_passed = 0 + + for test_case in subtask["testcases"]: + if results_dict[test_case]["correct"]: + subtask_passed += 1 + + # Handle min-score grading + if "grading" in subtask and subtask["grading"] == "min-score": + test_scores = [ + results_dict[test_case].get("score", 0) for test_case in subtask["testcases"] + ] + subtask_score = subtask["score"] * min(test_scores) + + # Full score if all tests passed + if subtask_passed == len(subtask["testcases"]): + subtask_score = subtask["score"] + + score["subtasks"][i] = { + "testcases": len(subtask["testcases"]), + "score": subtask_score, + "passed": subtask_passed / len(subtask["testcases"]), + } + + score["score"] = round(sum([subtask["score"] for subtask in score["subtasks"].values()])) + score["tests_passed"] = len(test_cases_passed) / len(results) if results else 0 + score["ace"] = score["tests_passed"] == 1 + + return score + + +def total_points(subtasks: Mapping[str, Mapping[str, Any]]) -> int: + """The problem's maximum score — upstream ``Problem.get_total_points``.""" + return sum(int(subtask["score"]) for subtask in subtasks.values()) diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index fa8d0d92..b5c5fb8c 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -141,6 +141,10 @@ from .livecodebench_code_generation import ( LiveCodeBenchDataset, LiveCodeBenchDatasetSample, ) +from .liveoibench import ( + LiveOIBenchDataset, + LiveOIBenchDatasetSample, +) from .math_500 import ( MATH500Dataset, MATH500DatasetSample, @@ -317,6 +321,8 @@ __all__ = [ "InverseIFEvalDatasetSample", "LiveCodeBenchDataset", "LiveCodeBenchDatasetSample", + "LiveOIBenchDataset", + "LiveOIBenchDatasetSample", "MATH500Dataset", "MATH500DatasetSample", "MATHPerturbHardDataset", diff --git a/sieval/datasets/liveoibench.py b/sieval/datasets/liveoibench.py new file mode 100644 index 00000000..433d85de --- /dev/null +++ b/sieval/datasets/liveoibench.py @@ -0,0 +1,263 @@ +"""LiveOIBench dataset — informatics-olympiad problems with official test data. + +403 problems from 72 contests across 14 olympiads (2023–2025), published as three +HuggingFace repos that this dataset stages together: + +* problems and metadata (920 KB) — statements, limits, grader/starter bundles; +* contestant results (559 KB) — per-contest human rankings and medal cutoffs, + kept on :attr:`contestants_path` for the task's report to rank against; +* official test cases (**33.5 GB**) — one parquet per year, each a single row + group, so there is no cheap per-problem read out of them. + +Because of that last point the test corpus is **materialized once** into +per-problem directories — the layout upstream's ``process_dataset.py`` produces, +so an existing ``$LIVEOIBENCH_ROOT/data`` tree can be pointed at directly. Rows +carry the path rather than the payload: at ~83 MB of tests per problem, a column +holding them would put the whole corpus through the record pipeline. + +The ``subtasks`` column is joined in from the test parquets, where it lives; it +is ~20 KB per year, so column projection reads it without touching ``tests``. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import os +from typing import TypedDict, override + +import pyarrow.parquet as pq +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +# `load_subtasks` / `load_code_bundle` are ports of upstream's own payload +# decoding and live in community/ next to the rest of it; the task reads the same +# two functions from there rather than reaching into this module. + +LIVEOIBENCH_REVISION = "0e87d7c7a667979e00df41889656d73f1d3ce153" +LIVEOIBENCH_TESTS_REVISION = "05731838fc460d3235c70506ee177aeb68a610cf" +LIVEOIBENCH_CONTESTANTS_REVISION = "fdcc2ba59670b06ec55ee61bc771ed855ba6dba6" + +PROBLEMS_FILE = os.path.join("data", "liveoibench_v1.parquet") +CONTESTANTS_FILE = os.path.join("data", "contest_results.parquet") +TEST_YEARS = ("2023", "2024", "2025") + + +def year_parquet_name(year: str) -> str: + """The test parquet for one year. + + Deliberately not spelled ``test_*``: pytest collects any imported callable + with that prefix as a test case, so a test module using this helper would + fail to collect on the helper itself. + """ + return f"liveoibench_testcases_v1_{year}.parquet" + + +def parse_problem_id(problem_id: str) -> tuple[str, str, str, str]: + """Split ``competition-year-round-task``; upstream ``parse_problem_id``. + + The task token is everything after the third segment, so a task name that + itself contains ``-`` (``grading-server``) survives. It is the same value the + ``task_name`` column carries, and it is what the prompt's code fence and the + extractor's first pattern are built from. + """ + parts = problem_id.split("-") + if len(parts) < 4: + raise ValueError( + f"Invalid problem_id {problem_id!r}; expected at least 4 tokens." + ) + return parts[0], parts[1], parts[2], "-".join(parts[3:]) + + +def problem_tests_dir(tests_root: str, problem_id: str) -> str: + """Where one problem's ``.in`` / ``.out`` files live once materialized. + + Upstream's reconstructed layout + (``{competition}/{year}/{round}/{task}/tests``), so a tree already built by + ``process_dataset.py`` can be used as *tests_root* unchanged. + """ + competition, year, round_name, task = parse_problem_id(problem_id) + return os.path.join(tests_root, competition, year, round_name, task, "tests") + + +class LiveOIBenchDatasetSample(TypedDict): + problem_id: str + competition: str + contest: str + task_name: str + problem_statement: str + time_limit: float + memory_limit: float + task_type: str + difficulty: int + algorithms: str + grader_code: str + starter_code: str + # Joined from the test parquet: `{id: {"score": int, "testcases": [name]}}`. + subtasks: str + # Derived. `contest_id` is the join key into the contestant table, whose own + # column is spelled that way; `tests_dir` is where this problem's test files + # were materialized. + contest_id: str + tests_dir: str + + +@sieval_dataset( + name="liveoibench", + display_name="LiveOIBench", + description=( + "403 olympiad problems with official tests, subtask rubrics and human " + "rankings; 33.5 GB of tests." + ), + source=( + f"hf:LiveOIBench/LiveOIBench@{LIVEOIBENCH_REVISION}", + f"hf:LiveOIBench/LiveOIBench_tests@{LIVEOIBENCH_TESTS_REVISION}", + f"hf:LiveOIBench/LiveOIBench_contestants@{LIVEOIBENCH_CONTESTANTS_REVISION}", + ), + categories=(Category(Level1Category.CODE, "CodeGeneration"),), + tags=("english", "cpp", "code-exec"), + license="cc-by-4.0", +) +class LiveOIBenchDataset(Dataset[LiveOIBenchDatasetSample]): + @override + def load( + self, + name_or_path: str, + task_type: str | None = "batch", + year: str | None = None, + competition: str | None = None, + tests_root: str | None = None, + require_tests: bool = True, + **kwargs, + ) -> HFDatasetDict: + """Load the problems, joined with their subtasks and test locations. + + *task_type* defaults to ``"batch"`` — the 380 problems the evaluator's + C++ path can grade. The 23 ``interactive`` ones need an interactor + process it does not run yet, and are filtered out rather than scored + zero, which would understate every model invisibly. Pass ``None`` to keep + every problem. + + *tests_root* defaults to a ``tests/`` tree beside the staged parquet + files; point it at an existing ``$LIVEOIBENCH_ROOT/data`` to reuse one. + *require_tests* is the guard that turns a missing corpus into an + actionable error instead of a run that fails 380 times. + """ + staged_root = os.path.dirname(os.path.normpath(name_or_path)) + problems_path = os.path.join(name_or_path, PROBLEMS_FILE) + tests_repo = os.path.join(staged_root, "LiveOIBench_tests") + contestants_repo = os.path.join(staged_root, "LiveOIBench_contestants") + + # Captured for the task, the way SciCode captures its h5 path; + # copy.copy-based clones (slice/shuffle/...) preserve the attribute. + self._contestants_path = os.path.join(contestants_repo, CONTESTANTS_FILE) + self._tests_root = tests_root or os.path.join(tests_repo, "materialized") + + table = pq.read_table(problems_path) + rows: list[dict] = table.to_pylist() + + subtasks_by_id = _read_subtasks(tests_repo) + kept: list[dict] = [] + for row in rows: + problem_id = row["problem_id"] + if task_type is not None and row["task_type"] != task_type: + continue + competition_name, problem_year, round_name, _task = parse_problem_id( + problem_id + ) + if year is not None and problem_year != year: + continue + if competition is not None and competition_name != competition: + continue + subtasks = subtasks_by_id.get(problem_id) + if subtasks is None: + # A problem with no subtask rubric has no score to compute; that + # is a broken join, not an empty result. + raise ValueError( + f"No subtasks found for {problem_id!r} in {tests_repo!r}; " + "the problem and test parquets are out of step." + ) + kept.append( + row + | { + "subtasks": subtasks, + "contest_id": "-".join( + (competition_name, problem_year, round_name) + ), + "tests_dir": problem_tests_dir(self._tests_root, problem_id), + } + ) + + if not kept: + raise ValueError( + "LiveOIBench dataset is empty after filtering " + f"(task_type={task_type!r}, year={year!r}, " + f"competition={competition!r})." + ) + + if require_tests: + missing = [ + row["problem_id"] for row in kept if not os.path.isdir(row["tests_dir"]) + ] + if missing: + raise FileNotFoundError( + f"{len(missing)} of {len(kept)} problems have no materialized " + f"test directory under {self._tests_root!r} " + f"(e.g. {missing[0]!r}). The 33.5 GB test corpus ships as " + "three single-row-group parquets and has to be unpacked " + "once:\n" + " python scripts/materialize_liveoibench_tests.py\n" + "Pass tests_root= to reuse a tree built by upstream's " + "process_dataset.py, or require_tests=False to load prompts " + "without grading." + ) + + return ensure_dataset_dict(HFDatasetDict({"test": HFDataset.from_list(kept)})) + + @property + def contestants_path(self) -> str | None: + """Staged path of the contestant-results parquet, for ranking a model. + + ``None`` when the dataset was built from a pre-loaded dict rather than + via :meth:`load`. + """ + return getattr(self, "_contestants_path", None) + + @property + def tests_root(self) -> str | None: + """Root of the materialized test tree; ``None`` as for + :attr:`contestants_path`.""" + return getattr(self, "_tests_root", None) + + +def _read_subtasks(tests_repo: str) -> dict[str, str]: + """Read every year's ``subtasks`` column, skipping the ``tests`` column. + + Each year file is one row group whose ``tests`` column is 6–17 GB and whose + ``subtasks`` column is ~20 KB, so projecting to two columns is the difference + between a 20 KB read and the whole corpus. + """ + subtasks: dict[str, str] = {} + for year in TEST_YEARS: + path = os.path.join(tests_repo, year_parquet_name(year)) + if not os.path.exists(path): + continue + table = pq.read_table(path, columns=["problem_id", "subtasks"]) + for problem_id, payload in zip( + table.column("problem_id").to_pylist(), + table.column("subtasks").to_pylist(), + strict=True, + ): + subtasks[problem_id] = payload or "{}" + if not subtasks: + raise FileNotFoundError( + f"No LiveOIBench test parquet found under {tests_repo!r}; " + "run `sieval dataset download liveoibench` first." + ) + return subtasks diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 5a5296b5..e412f5e3 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -821,6 +821,30 @@ "license": "cc", "checksums": {} }, + { + "name": "liveoibench", + "display_name": "LiveOIBench", + "description": "403 olympiad problems with official tests, subtask rubrics and human rankings; 33.5 GB of tests.", + "source": [ + "hf:LiveOIBench/LiveOIBench@0e87d7c7a667979e00df41889656d73f1d3ce153", + "hf:LiveOIBench/LiveOIBench_tests@05731838fc460d3235c70506ee177aeb68a610cf", + "hf:LiveOIBench/LiveOIBench_contestants@fdcc2ba59670b06ec55ee61bc771ed855ba6dba6" + ], + "categories": [ + { + "level1": "Code", + "level2": "CodeGeneration" + } + ], + "tags": [ + "english", + "cpp", + "code-exec" + ], + "deps_group": null, + "license": "cc-by-4.0", + "checksums": {} + }, { "name": "math_500", "display_name": "MATH-500", @@ -2369,6 +2393,28 @@ "status": "stable", "reference_kind": "procedure" }, + { + "name": "liveoibench_0shot_gen", + "display_name": "LiveOIBench (0-shot)", + "description": "LiveOIBench — olympiad C++ problems, subtask-scored and ranked against human contestants.", + "dataset": "liveoibench", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "cpp", + "code-exec" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "liveoibench", + "url": "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/base_judge.py", + "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Codeforces Elo is not computed." + }, + "status": "stable", + "reference_kind": "procedure" + }, { "name": "math_500_0shot_gen", "display_name": "MATH-500 (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 79df62e8..dc8656f4 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -130,6 +130,9 @@ from .livecodebench_code_generation_0shot_gen import ( from .livecodebench_code_generation_kshot_base_gen import ( LiveCodeBenchCodeGenerationFewShotBaseGenTask, ) +from .liveoibench_0shot_gen import ( + LiveOIBenchZeroShotGenTask, +) from .math_500_0shot_gen import ( MATH500ZeroShotGenTask, ) @@ -283,6 +286,7 @@ __all__ = [ "InverseIFEvalZeroShotGenTask", "LiveCodeBenchCodeGenerationFewShotBaseGenTask", "LiveCodeBenchCodeGenerationZeroShotGenTask", + "LiveOIBenchZeroShotGenTask", "MATH500ZeroShotGenTask", "MATHPerturbHardZeroShotGenTask", "MATHPerturbSimpleZeroShotGenTask", diff --git a/sieval/tasks/liveoibench_0shot_gen.py b/sieval/tasks/liveoibench_0shot_gen.py new file mode 100644 index 00000000..84362906 --- /dev/null +++ b/sieval/tasks/liveoibench_0shot_gen.py @@ -0,0 +1,491 @@ +"""LiveOIBench — informatics-olympiad problems, graded in C++ with subtask credit. + +Grading runs in the evaluator service (``source="liveoibench"``), which compiles +the submission with ``g++`` and runs every official test case under the +problem's own time and memory limits. Two things make this task unlike the other +code benchmarks here: + +* **Partial credit.** A submission is not pass/fail — a subtask pays its points + only when every test in it passes, and a problem's score is the sum. The + evaluator therefore returns the whole verdict vector rather than a count. +* **A human baseline.** Every contest ships its real contestants' scores, so the + report ranks the model inside the contest it was actually sitting. + +Test data is read by the evaluator from the materialized corpus by path, not +shipped inline: a problem averages ~140 MB of test cases, which would otherwise +cross the wire once per rollout. Pass ``inline_tests=True`` when the evaluator +cannot see that volume. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import json +import os +import time +from collections import defaultdict +from typing import override + +import httpx +from loguru import logger + +from sieval.community.liveoibench.code_extractor import CodeExtractor +from sieval.community.liveoibench.payloads import load_code_bundle, load_subtasks +from sieval.community.liveoibench.prompts import build_prompt +from sieval.community.liveoibench.rankings import score_contest +from sieval.community.liveoibench.scoring import interprete_task_result, total_points +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + NonRetriableSampleError, + PredictionRecord, + PromptRecord, + ReferenceImpl, + RolloutJudgement, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, + budget_metrics, + health_metrics, +) +from sieval.datasets import LiveOIBenchDatasetSample + +_UPSTREAM = "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12" + + +@sieval_task( + name="liveoibench_0shot_gen", + display_name="LiveOIBench (0-shot)", + description=( + "LiveOIBench — olympiad C++ problems, subtask-scored and ranked against " + "human contestants." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "cpp", "code-exec"), + model_type="chat", + reference_kind="procedure", + reference_impl=ReferenceImpl( + source="liveoibench", + url=f"{_UPSTREAM}/src/judges/base_judge.py", + notes=( + "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), " + "subtask scoring (base_judge.py interprete_task_result) and ranking " + "(generate_rankings.py) are ported under sieval/community/liveoibench/. " + "Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe " + "-static -g, " + "RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, " + "and its output comparison. No checker path: the published dataset ships " + "no checkers/ directory, so upstream's own judge compares outputs directly " + "and the 36 min-score subtasks collapse onto the all-or-nothing rule. " + "Upstream samples n=8 per problem and reports the best; this task defaults " + "to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 " + "interactive ones are filtered out by the dataset (no interactor process " + "yet), so numbers are over 380 problems and are NOT comparable to the " + "paper's 403-problem table. Codeforces Elo is not computed." + ), + ), +) +class LiveOIBenchZeroShotGenTask( + Task[ + LiveOIBenchDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + # `float | str`: the report carries `score_key`, which names a column + # rather than measuring one. + dict[str, float | str], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + n: int = 1, + max_concurrency: int = 4, + inline_tests: bool = False, + ): + """*n* is the sampling budget; upstream draws 8 and reports the best of + them, which this task reproduces for whatever *n* it is given. + + *inline_tests* ships each problem's test cases in the request body + instead of naming the directory. Only for an evaluator that cannot read + the materialized corpus — it moves ~140 MB per rollout on average. + """ + super().__init__(dataset=dataset, model=model, name=name) + self._n = n + self._inline_tests = inline_tests + self._code_eval_api = os.getenv( + "SIEVAL_CODE_EVAL_API", "http://localhost:11451/evaluations" + ) + self._http_client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=max_concurrency) + ) + + @override + async def preprocess(self, raw, ctx): + prompt = build_prompt( + task=raw["task_name"], + statement=raw["problem_statement"], + time_limit=raw["time_limit"], + memory_limit=raw["memory_limit"], + starter_codes=load_code_bundle(raw["starter_code"]), + ) + # No system message: upstream's default run sends the prompt alone + # (`--system_prompt` is opt-in and supplies a per-model string). + return build_prompt_record( + [{"role": "user", "content": prompt}], + # No `reference`: the ground truth is a test suite and a subtask + # rubric, described in the judgement's extra instead. + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + task_name = ctx.raw_sample["task_name"] + # `extract_code` returns ("", 'empty') for an empty fence and + # (None, 'not_found') for no fence; both normalize to None so + # `extracted` reports a miss rather than an empty submission. + return build_prediction_record( + [ + CodeExtractor.extract_code(choice, task_name)[0] or None + for choice in inf.texts + ] + ) + + @override + async def feedback(self, post, ctx): + raw = ctx.raw_sample + subtasks = load_subtasks(raw["subtasks"]) + if not subtasks: + # Without a rubric there is no score to compute, and re-inferring + # cannot produce one. + raise NonRetriableSampleError( + f"{raw['problem_id']}: no subtasks to score against" + ) + maximum = total_points(subtasks) + payload_tests = self._test_payload(raw) + + rollouts: list[RolloutJudgement] = [] + best_index, best_key = None, None + for rollout in post["rollouts"]: + idx = rollout["index"] + verdicts, case_names, msg, resources = await self._grade( + raw, rollout.get("prediction") or "", payload_tests, idx + ) + + results = [ + {"test_case": name, "correct": correct} + for name, correct in zip(case_names, verdicts, strict=True) + ] + try: + scored = interprete_task_result(results, subtasks) + except KeyError as e: + # A subtask naming a test the corpus does not contain: the two + # parquets disagree, and no rollout of this sample can be scored. + raise NonRetriableSampleError( + f"{raw['problem_id']}: subtask references unknown test case {e}" + ) from e + + relative_score = (scored["score"] / maximum * 100) if maximum else 0.0 + rollouts.append( + build_rollout_judgement( + idx, + # `correct` is the cross-task axis, so it is the olympiad's + # own all-or-nothing reading: every test passed. + scored["ace"], + score=float(scored["score"]), + metrics={ + "relative_score": relative_score, + "tests_passed_pct": scored["tests_passed"] * 100, + "ace": scored["ace"], + }, + extra={ + "msg": msg, + "subtasks": scored["subtasks"], + "n_cases": len(case_names), + "n_passed": sum(verdicts), + "resources": resources, + }, + ) + ) + # Upstream ranks candidates by score, then relative_score, then + # tests_passed_pct. Within one problem the first two are the same + # ordering, so this is (score, tests_passed). Upstream's final + # tiebreak is execution time, which is not recorded here; the lowest + # rollout index wins instead, so the pick is deterministic. + key = (scored["score"], scored["tests_passed"]) + if best_key is None or key > best_key: + best_index, best_key = idx, key + + best = next(r for r in rollouts if r["index"] == best_index) + return True, build_judgement_record( + None, # the reference is a test suite plus a rubric, not a value + rollouts, + score=best["score"], + metrics={ + "relative_score": best["metrics"]["relative_score"], + "tests_passed_pct": best["metrics"]["tests_passed_pct"], + "ace": best["metrics"]["ace"], + }, + extra={ + "problem_id": raw["problem_id"], + "contest_id": raw["contest_id"], + "task_name": raw["task_name"], + "total_points": maximum, + "n_subtasks": len(subtasks), + "best_rollout": best_index, + "task_type": raw["task_type"], + }, + ) + + def _test_payload(self, raw) -> dict: + """The request's test half: a directory to read, or the cases inline.""" + tests_dir = raw["tests_dir"] + if not self._inline_tests: + return {"test_dir": tests_dir} + + inputs, outputs, names = [], [], [] + for filename in sorted(f for f in os.listdir(tests_dir) if f.endswith(".in")): + stem = filename[: -len(".in")] + output_path = os.path.join(tests_dir, f"{stem}.out") + if not os.path.exists(output_path): + raise NonRetriableSampleError( + f"{raw['problem_id']}: test {stem!r} has no .out in {tests_dir}" + ) + with open( + os.path.join(tests_dir, filename), encoding="utf-8", errors="replace" + ) as f: + inputs.append(f.read()) + with open(output_path, encoding="utf-8", errors="replace") as f: + outputs.append(f.read()) + names.append(stem) + return {"test": {"inputs": inputs, "outputs": outputs, "names": names}} + + async def _grade( + self, raw, code: str, payload_tests: dict, idx: int + ) -> tuple[list[bool], list[str], str, dict]: + """One rollout through the evaluator; returns verdicts and their names.""" + task_name = raw["task_name"] + # Compiled with the submission when the problem ships one; the header has + # to be on disk for the submission's own `#include` to resolve. + files = load_code_bundle(raw["grader_code"]) + + try: + resp = await self._http_client.post( + self._code_eval_api, + json={ + "uuid": f"{raw['problem_id']}-{idx}-{time.perf_counter_ns()}", + "source": "liveoibench", + "lang": "cpp", + # An unextractable answer is "" on the wire, so the evaluator + # still runs it and reports a compile error -- a real verdict + # rather than a skipped rollout. + "code": code, + "entry_filename": f"{task_name}.cpp", + "files": files, + # The problem's own limit, per case; the evaluator applies + # upstream's 20% buffer on top. + "timeout_per_case": float(raw["time_limit"]), + "memory_limit": int(raw["memory_limit"]), + **payload_tests, + }, + # Every case is bounded, so this only has to outlast the whole + # suite: the per-case wall the evaluator enforces is 120s, and a + # problem can carry a few hundred cases. + timeout=self._request_timeout(raw), + ) + resp.raise_for_status() + res = resp.json() + msg = res["msg"] + data = res["data"] or {} + except Exception as e: + logger.warning( + "Evaluation error for {} rollout {}: [{}] {}", + raw["problem_id"], + idx, + type(e).__name__, + e, + ) + raise + + verdicts = data.get("case_verdicts") + case_names = data.get("case_names") + if verdicts is None or case_names is None: + # Reached an evaluator without the C++ path (or an older one): there + # is no verdict vector to score subtasks from, and `status` alone + # would silently score every partial solution as zero. + raise RuntimeError( + f"evaluator returned no per-case verdicts for {raw['problem_id']!r} " + f"(msg: {msg}); it needs the liveoibench C++ source " + "(vendor/code-evaluator/VENDORED.md)." + ) + + resources = { + key: value + for key, value in data.items() + if key not in ("n_cases", "n_passed", "case_verdicts", "case_names") + } + return verdicts, case_names, msg, resources + + def _request_timeout(self, raw) -> float: + """HTTP deadline, outside every budget the evaluator can spend. + + Compilation is bounded at the evaluator's 60s default and each case at + its own 120s wall, which only a process burning no CPU can reach. The + suite runs at most 4 cases at a time there, so the bound is derived from + the case count rather than assumed. + """ + try: + n_cases = sum(1 for f in os.listdir(raw["tests_dir"]) if f.endswith(".in")) + except OSError: + n_cases = 100 + per_case = min(120.0, float(raw["time_limit"]) * 1.2 + 5.0) + return 60.0 + per_case * n_cases + 30.0 + + @override + async def report(self, finals, fails): + total = len(finals) + len(fails) + if not finals: + return { + "score": 0.0, + "fails": float(len(fails)), + "relative_score": 0.0, + SCORE_KEY_FIELD: "relative_score", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + + # A failed sample scores 0 rather than dropping out of the mean, so the + # denominator is every sample requested. + relative = ( + sum(f.feedback_result["metrics"]["relative_score"] for f in finals) / total + ) + pass_rate = ( + sum(f.feedback_result["metrics"]["tests_passed_pct"] for f in finals) + / total + ) + ace_rate = ( + sum(1 for f in finals if f.feedback_result["metrics"]["ace"]) / total * 100 + ) + + metrics: dict[str, float | str] = { + "score": relative, + "fails": float(len(fails)), + "relative_score": relative, + "pass_rate": pass_rate, + "ace_rate": ace_rate, + "n_problems": float(len(finals)), + SCORE_KEY_FIELD: "relative_score", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + metrics |= self._human_metrics(finals) + metrics |= budget_metrics( + [f.feedback_result["n_rollouts"] for f in finals], n=self._n, k=self._n + ) + return metrics | health_metrics(finals) + + def _human_metrics(self, finals) -> dict[str, float]: + """Percentile and medals, per contest, against that contest's field. + + Upstream re-totals the humans over exactly the tasks the model was scored + on, so a contest whose interactive problems were filtered out still + compares like with like. Contests are averaged unweighted, as upstream + averages its per-contest percentiles. + """ + contestants = self._load_contestants() + if not contestants: + return {} + + by_contest: dict[str, dict[str, float]] = defaultdict(dict) + for final in finals: + extra = final.feedback_result["extra"] + by_contest[extra["contest_id"]][extra["task_name"]] = final.feedback_result[ + "score" + ] + + percentiles: list[float] = [] + medals: list[str] = [] + unmatched = 0 + for contest_id, model_scores in by_contest.items(): + contest = contestants.get(contest_id) + if contest is None: + unmatched += 1 + continue + result = score_contest( + contest["rankings"], + model_scores, + contest_id=contest_id, + gold_cutoff=contest["gold_cutoff"], + silver_cutoff=contest["silver_cutoff"], + bronze_cutoff=contest["bronze_cutoff"], + ) + if result["human_percentile"] is not None: + percentiles.append(result["human_percentile"]) + if result["medal"] is not None: + medals.append(result["medal"]) + + if unmatched: + logger.warning( + "{} contest(s) had no contestant record; excluded from the " + "human metrics.", + unmatched, + ) + + metrics: dict[str, float] = {"n_contests": float(len(by_contest))} + if percentiles: + metrics["human_percentile"] = sum(percentiles) / len(percentiles) + metrics["n_contests_ranked"] = float(len(percentiles)) + if medals: + metrics["medal_rate"] = ( + sum(1 for m in medals if m in ("Gold", "Silver", "Bronze")) + / len(medals) + * 100 + ) + metrics["gold_rate"] = ( + sum(1 for m in medals if m == "Gold") / len(medals) * 100 + ) + return metrics + + def _load_contestants(self) -> dict[str, dict]: + """The contestant table, keyed by contest, or ``{}`` when unavailable.""" + path = getattr(self.dataset, "contestants_path", None) + if not path or not os.path.exists(path): + logger.warning( + "No contestant results at {}; reporting scores without human ranking.", + path, + ) + return {} + + import pyarrow.parquet as pq + + table = pq.read_table(path) + contestants: dict[str, dict] = {} + for row in table.to_pylist(): + try: + rankings = json.loads(row["contestants_ranking"] or "[]") + except json.JSONDecodeError: + continue + contestants[row["contest_id"]] = { + "rankings": rankings, + "gold_cutoff": row["gold_cutoff"], + "silver_cutoff": row["silver_cutoff"], + "bronze_cutoff": row["bronze_cutoff"], + } + return contestants + + @override + async def shutdown(self): + await self._http_client.aclose() diff --git a/tests/unit/community/liveoibench/__init__.py b/tests/unit/community/liveoibench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/community/liveoibench/test_prompts.py b/tests/unit/community/liveoibench/test_prompts.py new file mode 100644 index 00000000..7b32fde4 --- /dev/null +++ b/tests/unit/community/liveoibench/test_prompts.py @@ -0,0 +1,80 @@ +"""Unit tests for LiveOIBench prompt construction. + +The prompt is the benchmark's input, so these pin the exact strings upstream +assembles -- including the two things a tidy-up would change first: the limits +rendering as the floats the parquet carries, and the grader block appearing only +when all three attachment files are present. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +from sieval.community.liveoibench.prompts import build_prompt + +STATEMENT = "# Beech Tree\n\nGiven $N$ nodes...\n" + + +def test_the_fence_is_named_after_the_task(): + prompt = build_prompt("beechtree", STATEMENT, 1.5, 2048.0) + assert prompt.startswith( + "Given a competition problem below, write a solution in C++ that solves " + "all the subtasks. Make sure to wrap your code in '```beechtree.cpp' and " + "'```' Markdown delimiters.\n\n" + ) + + +def test_the_statement_is_delimited_verbatim(): + prompt = build_prompt("t", STATEMENT, 1.0, 256.0) + assert f"[BEGIN PROBLEM]\n{STATEMENT}[END PROBLEM]\n" in prompt + + +def test_limits_render_as_the_floats_the_dataset_carries(): + # Upstream interpolates the parquet value straight in, so 2048.0 -- not 2048 + # -- is what every published run showed the model. + prompt = build_prompt("t", STATEMENT, 1.5, 2048.0) + assert "Time limit: 1.5 seconds\nMemory limit: 2048.0 MB\n" in prompt + + +def test_absent_limits_render_as_unknown(): + prompt = build_prompt("t", STATEMENT, None, None) + assert "Time limit: unknown seconds\nMemory limit: unknown MB\n" in prompt + + +def test_without_attachments_the_plain_instruction_is_appended(): + prompt = build_prompt("t", STATEMENT, 1.0, 256.0) + assert prompt.endswith( + "Generate a solution in C++ that solves the task. Make sure to wrap your " + "code in '```t.cpp' and '```' Markdown delimiters.\n\n" + ) + assert "grader.cpp" not in prompt + + +def test_the_grader_block_needs_all_three_files(): + partial = {"grader.cpp": "int main(){}\n", "t.h": "void f();\n"} # no t.cpp + assert "```grader.cpp" not in build_prompt("t", STATEMENT, 1.0, 256.0, partial) + + whole = partial | {"t.cpp": "void f(){}\n"} + prompt = build_prompt("t", STATEMENT, 1.0, 256.0, whole) + assert ( + "We are going to grade your solution using the following grader.cpp" in prompt + ) + assert "```grader.cpp\nint main(){}\n```\n\n" in prompt + assert "```t.h\nvoid f();\n```\n\n" in prompt + assert "```t.cpp\nvoid f(){}\n```\n\n" in prompt + + +def test_stub_cpp_stands_in_for_grader_cpp(): + # Upstream falls back to `stub.cpp` when a problem ships no `grader.cpp`, and + # still labels the block `grader.cpp`. + bundle = { + "stub.cpp": "int main(){}\n", + "t.h": "void f();\n", + "t.cpp": "void f(){}\n", + } + prompt = build_prompt("t", STATEMENT, 1.0, 256.0, bundle) + assert "```grader.cpp\nint main(){}\n```\n\n" in prompt + + +def test_a_task_name_with_a_dash_survives_into_the_fence(): + # `CEOI-2023-contest-grading-server` -> task `grading-server`. + prompt = build_prompt("grading-server", STATEMENT, 1.0, 256.0) + assert "'```grading-server.cpp'" in prompt diff --git a/tests/unit/community/liveoibench/test_rankings.py b/tests/unit/community/liveoibench/test_rankings.py new file mode 100644 index 00000000..8edb3f16 --- /dev/null +++ b/tests/unit/community/liveoibench/test_rankings.py @@ -0,0 +1,146 @@ +"""Unit tests for ranking a model against a contest's human contestants. + +The load-bearing behaviour is that humans are re-totaled over **only** the tasks +the model was scored on: sieval filters interactive problems out, so a contest is +often partial, and comparing a partial model total against full human totals +would understate every model by however much it skipped. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +from sieval.community.liveoibench.rankings import ( + calculate_percentile, + identify_task_columns, + medal_from_cutoffs, + normalize_name, + score_contest, +) + +# Shape of the published `contestants_ranking` payload. +RANKINGS = [ + { + "Rank": 1, + "Contestant": "A", + "Country": "CN", + "alpha": 100, + "beta": 100, + "Total": 200, + }, + { + "Rank": 2, + "Contestant": "B", + "Country": "US", + "alpha": 100, + "beta": 20, + "Total": 120, + }, + { + "Rank": 3, + "Contestant": "C", + "Country": "PL", + "alpha": 40, + "beta": 0, + "Total": 40, + }, + {"Rank": 4, "Contestant": "D", "Country": "JP", "alpha": 0, "beta": 0, "Total": 0}, +] + + +def test_percentile_counts_only_contestants_strictly_below(): + # Upstream is `model_score > human_scores`, so tying the field beats nobody. + assert calculate_percentile(40, [0, 40, 120, 200]) == 25.0 + assert calculate_percentile(0, [0, 40, 120, 200]) == 0.0 + assert calculate_percentile(201, [0, 40, 120, 200]) == 100.0 + + +def test_percentile_of_an_empty_field_is_unknown_not_zero(): + assert calculate_percentile(100, []) is None + + +def test_medal_ladder_is_inclusive_at_each_cutoff(): + assert medal_from_cutoffs(236, 236, 191, 145) == "Gold" + assert medal_from_cutoffs(235, 236, 191, 145) == "Silver" + assert medal_from_cutoffs(145, 236, 191, 145) == "Bronze" + assert medal_from_cutoffs(144, 236, 191, 145) == "None" + + +def test_a_contest_publishing_no_cutoffs_yields_no_medal_rather_than_none_the_string(): + assert medal_from_cutoffs(500, None, None, None) is None + + +def test_header_columns_are_not_mistaken_for_tasks(): + mapping = identify_task_columns(list(RANKINGS[0])) + assert set(mapping) == {"alpha", "beta"} + + +def test_boi_scores_by_day_so_day_columns_are_tasks_there(): + columns = ["Rank", "Contestant", "Day1", "Day2"] + assert identify_task_columns(columns, "BOI-2025-contest") == { + "day1": "Day1", + "day2": "Day2", + } + assert identify_task_columns(columns, "IOI-2025-contest") == {} + + +def test_normalize_name_strips_everything_but_alphanumerics(): + assert normalize_name("Cow-libi (Silver)") == "cowlibisilver" + + +def test_humans_are_retotaled_over_only_the_tasks_the_model_attempted(): + # Model scored 100 on `alpha` alone. Against alpha-only human totals + # (100, 100, 40, 0) it beats two; against full totals it would beat one. + result = score_contest(RANKINGS, {"alpha": 100}) + assert result["model_total"] == 100 + assert result["matched_columns"] == ["alpha"] + assert result["human_percentile"] == 50.0 + + +def test_scoring_every_task_uses_every_column(): + result = score_contest(RANKINGS, {"alpha": 100, "beta": 100}) + assert result["model_total"] == 200 + assert sorted(result["matched_columns"]) == ["alpha", "beta"] + # Ties with the top contestant, so it beats the other three. + assert result["human_percentile"] == 75.0 + + +def test_task_names_match_columns_case_and_punctuation_insensitively(): + rankings = [{"Rank": 1, "Cow-libi": 100, "Total": 100}] + result = score_contest(rankings, {"cow_libi": 50}) + assert result["matched_columns"] == ["Cow-libi"] + + +def test_unmatched_task_names_fall_back_to_the_contest_total(): + result = score_contest(RANKINGS, {"gamma": 130}) + assert result["matched_columns"] == ["Total"] + assert result["model_total"] == 130 + # Against Total (200, 120, 40, 0) a 130 beats three. + assert result["human_percentile"] == 75.0 + + +def test_a_field_with_no_usable_column_at_all_reports_no_percentile(): + result = score_contest([{"Rank": 1}], {"gamma": 10}) + assert result["human_percentile"] is None + assert result["model_total"] == 10 + + +def test_non_numeric_and_missing_cells_count_as_zero(): + rankings = [ + {"alpha": "-"}, # withdrew + {"alpha": None}, + {}, # column absent for this row + {"alpha": "50"}, # numeric strings still count + ] + result = score_contest(rankings, {"alpha": 10}) + assert result["human_percentile"] == 75.0 + + +def test_medals_use_the_contests_published_cutoffs(): + result = score_contest( + RANKINGS, + {"alpha": 100, "beta": 100}, + gold_cutoff=200.0, + silver_cutoff=120.0, + bronze_cutoff=40.0, + ) + assert result["medal"] == "Gold" + assert result["n_contestants"] == 4 diff --git a/tests/unit/community/liveoibench/test_scoring.py b/tests/unit/community/liveoibench/test_scoring.py new file mode 100644 index 00000000..77f4549f --- /dev/null +++ b/tests/unit/community/liveoibench/test_scoring.py @@ -0,0 +1,108 @@ +"""Unit tests for LiveOIBench subtask scoring. + +The arithmetic is the benchmark: a subtask is all-or-nothing, so a submission +that passes most tests of every subtask can score zero while another that passes +fewer tests scores points. Pinning that here keeps a "reasonable" partial-credit +rewrite from quietly becoming a different benchmark. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import pytest + +from sieval.community.liveoibench.scoring import interprete_task_result, total_points + +SUBTASKS = { + "1": {"task": "Subtask 1", "score": 0, "testcases": ["s0"]}, + "2": {"task": "Subtask 2", "score": 30, "testcases": ["a1", "a2"]}, + "3": {"task": "Subtask 3", "score": 70, "testcases": ["b1", "b2"]}, +} + + +def _results(**passed: bool) -> list[dict]: + return [{"test_case": name, "correct": ok} for name, ok in passed.items()] + + +def test_a_subtask_pays_only_when_every_test_in_it_passes(): + scored = interprete_task_result( + _results(s0=True, a1=True, a2=True, b1=True, b2=False), SUBTASKS + ) + # Subtask 3 lost one test of two, so its 70 points are gone entirely. + assert scored["score"] == 30 + assert scored["subtasks"]["3"]["score"] == 0 + assert scored["subtasks"]["3"]["passed"] == 0.5 + + +def test_more_tests_passed_can_still_score_less(): + # 4 of 5 tests, spread so that both paying subtasks are broken -> 0 points. + spread = interprete_task_result( + _results(s0=True, a1=True, a2=False, b1=True, b2=True), SUBTASKS + ) + # 3 of 5 tests, but one whole subtask intact -> 30 points. + focused = interprete_task_result( + _results(s0=False, a1=True, a2=True, b1=True, b2=False), SUBTASKS + ) + assert spread["tests_passed"] > focused["tests_passed"] + assert spread["score"] == 70 # subtask 3 whole + assert focused["score"] == 30 + + +def test_ace_and_tests_passed_span_every_result(): + scored = interprete_task_result( + _results(s0=True, a1=True, a2=True, b1=True, b2=True), SUBTASKS + ) + assert scored["score"] == 100 + assert scored["tests_passed"] == 1 + assert scored["ace"] is True + + +def test_a_zero_point_sample_subtask_does_not_change_the_score(): + without = interprete_task_result( + _results(s0=False, a1=True, a2=True, b1=True, b2=True), SUBTASKS + ) + assert without["score"] == 100 + assert without["ace"] is False, "a failed sample test still breaks the ace flag" + + +def test_total_is_rounded_not_truncated(): + # min-score grading with a checker score of 0.5 on a 45-point subtask -> 22.5. + subtasks = {"1": {"score": 45, "grading": "min-score", "testcases": ["a", "b"]}} + results = [ + {"test_case": "a", "correct": False, "score": 0.5}, + {"test_case": "b", "correct": True, "score": 1.0}, + ] + assert interprete_task_result(results, subtasks)["score"] == 22 + + +def test_min_score_collapses_to_all_or_nothing_without_a_checker(): + # The published dataset ships no checkers, so no result carries a `score` and + # upstream's `.get("score", 0)` reads 0 -- the same verdict as the default + # rule. Pinned because a port that invented per-test fractions here would + # silently pay partial credit upstream does not. + subtasks = {"1": {"score": 45, "grading": "min-score", "testcases": ["a", "b"]}} + partial = interprete_task_result( + [{"test_case": "a", "correct": True}, {"test_case": "b", "correct": False}], + subtasks, + ) + assert partial["score"] == 0 + whole = interprete_task_result( + [{"test_case": "a", "correct": True}, {"test_case": "b", "correct": True}], + subtasks, + ) + assert whole["score"] == 45 + + +def test_an_empty_result_set_scores_zero_rather_than_dividing_by_zero(): + scored = interprete_task_result([], {}) + assert scored == {"subtasks": {}, "score": 0, "tests_passed": 0, "ace": False} + + +def test_a_subtask_naming_an_unknown_test_raises(): + # The two parquets are out of step; scoring it would silently understate. + with pytest.raises(KeyError): + interprete_task_result([{"test_case": "a", "correct": True}], SUBTASKS) + + +def test_total_points_sums_the_rubric(): + assert total_points(SUBTASKS) == 100 + assert total_points({}) == 0 diff --git a/tests/unit/datasets/test_liveoibench.py b/tests/unit/datasets/test_liveoibench.py new file mode 100644 index 00000000..c1bda5df --- /dev/null +++ b/tests/unit/datasets/test_liveoibench.py @@ -0,0 +1,216 @@ +"""Unit tests for the LiveOIBench dataset loader. + +Covers the three-repo join (problems + subtasks + contestants), the derived +columns the task reads, and the two guards that turn a silent under-run into an +error: a missing test corpus and an empty filter. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import json +import os + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from sieval.datasets.liveoibench import ( + LiveOIBenchDataset, + parse_problem_id, + problem_tests_dir, + year_parquet_name, +) + +BATCH_ID = "IOI-2025-contest-beechtree" +INTERACTIVE_ID = "IOI-2025-contest-longesttrip" +DASHED_ID = "CEOI-2024-contest-grading-server" + +SUBTASKS = {"1": {"task": "Subtask 1", "score": 100, "testcases": ["t1"]}} + + +def _problem(problem_id: str, task_type: str = "batch") -> dict: + # The year is read out of `problem_id`, the way the loader reads it. + _, _, _, task = parse_problem_id(problem_id) + return { + "id": 0, + "problem_id": problem_id, + "competition": problem_id.split("-")[0], + "contest": "contest", + "task_name": task, + "problem_statement": f"# {task}\n", + "time_limit": 1.5, + "memory_limit": 2048.0, + "task_type": task_type, + "difficulty": 12, + "algorithms": '["greedy"]', + "setup_script": "", + "evaluation_script": "", + "grader_code": "", + "starter_code": "", + } + + +def _stage(root, problems: list[dict], *, with_tests: bool = True) -> str: + """Write the three staged repos the way `sieval dataset download` would.""" + problems_dir = root / "LiveOIBench" / "data" + problems_dir.mkdir(parents=True, exist_ok=True) + pq.write_table( + pa.Table.from_pylist(problems), problems_dir / "liveoibench_v1.parquet" + ) + + tests_repo = root / "LiveOIBench_tests" + tests_repo.mkdir(parents=True, exist_ok=True) + by_year: dict[str, list[dict]] = {} + for problem in problems: + year = parse_problem_id(problem["problem_id"])[1] + by_year.setdefault(year, []).append( + { + "id": 0, + "problem_id": problem["problem_id"], + "subtasks": json.dumps(SUBTASKS), + } + ) + for year, rows in by_year.items(): + pq.write_table(pa.Table.from_pylist(rows), tests_repo / year_parquet_name(year)) + + contestants_dir = root / "LiveOIBench_contestants" / "data" + contestants_dir.mkdir(parents=True, exist_ok=True) + pq.write_table( + pa.Table.from_pylist( + [ + { + "contest_id": "IOI-2025-contest", + "gold_cutoff": 90.0, + "silver_cutoff": 60.0, + "bronze_cutoff": 30.0, + "division": "Division 1", + "contestants_ranking": json.dumps([{"Rank": 1, "beechtree": 100}]), + "date": "2025-08-01", + "num_contestants": 1, + "problems": json.dumps([BATCH_ID]), + } + ] + ), + contestants_dir / "contest_results.parquet", + ) + + tests_root = tests_repo / "materialized" + if with_tests: + for problem in problems: + case_dir = problem_tests_dir(str(tests_root), problem["problem_id"]) + os.makedirs(case_dir, exist_ok=True) + with open(os.path.join(case_dir, "t1.in"), "w") as f: + f.write("1 2\n") + with open(os.path.join(case_dir, "t1.out"), "w") as f: + f.write("3\n") + return str(root / "LiveOIBench") + + +# --------------------------------------------------------------------------- # +# Identifier parsing +# --------------------------------------------------------------------------- # +def test_the_task_token_keeps_its_dashes(): + # `task_name` is everything after competition/year/round, so a hyphenated + # task name must not be truncated -- it names the prompt's code fence. + assert parse_problem_id(DASHED_ID) == ("CEOI", "2024", "contest", "grading-server") + + +def test_a_malformed_identifier_raises(): + with pytest.raises(ValueError, match="expected at least 4 tokens"): + parse_problem_id("IOI-2025-contest") + + +def test_tests_live_in_upstreams_reconstructed_layout(): + # Matching upstream means an existing `process_dataset.py` tree can be reused. + assert problem_tests_dir("/root", BATCH_ID) == ( + "/root/IOI/2025/contest/beechtree/tests" + ) + + +# --------------------------------------------------------------------------- # +# Loading and joining +# --------------------------------------------------------------------------- # +def test_load_joins_subtasks_and_derives_the_columns_the_task_reads(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + dataset = LiveOIBenchDataset(name_or_path=path) + + (row,) = list(dataset.test_set) + assert json.loads(row["subtasks"]) == SUBTASKS + assert row["contest_id"] == "IOI-2025-contest" + assert row["tests_dir"].endswith("IOI/2025/contest/beechtree/tests") + assert row["task_name"] == "beechtree" + + +def test_the_contestant_parquet_is_reachable_from_the_dataset(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + dataset = LiveOIBenchDataset(name_or_path=path) + assert dataset.contestants_path is not None + assert os.path.exists(dataset.contestants_path) + + +def test_interactive_problems_are_filtered_out_by_default(tmp_path): + # Filtered rather than scored zero: the evaluator has no interactor process, + # and a zero would understate every model without saying so. + path = _stage( + tmp_path, + [_problem(BATCH_ID), _problem(INTERACTIVE_ID, task_type="interactive")], + ) + assert [ + r["problem_id"] for r in LiveOIBenchDataset(name_or_path=path).test_set + ] == [BATCH_ID] + both = LiveOIBenchDataset(name_or_path=path, task_type=None).test_set + assert len(both) == 2 + + +def test_year_and_competition_filters_apply(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID), _problem(DASHED_ID)]) + by_year = LiveOIBenchDataset(name_or_path=path, year="2024").test_set + assert [r["problem_id"] for r in by_year] == [DASHED_ID] + by_competition = LiveOIBenchDataset(name_or_path=path, competition="IOI").test_set + assert [r["problem_id"] for r in by_competition] == [BATCH_ID] + + +def test_a_filter_matching_nothing_raises_instead_of_scoring_an_empty_set(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + with pytest.raises(ValueError, match="empty after filtering"): + LiveOIBenchDataset(name_or_path=path, year="1999") + + +def test_a_missing_test_corpus_names_the_command_that_builds_it(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)], with_tests=False) + with pytest.raises(FileNotFoundError, match="materialize_liveoibench_tests"): + LiveOIBenchDataset(name_or_path=path) + + +def test_prompts_can_be_loaded_without_the_corpus(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)], with_tests=False) + dataset = LiveOIBenchDataset(name_or_path=path, require_tests=False) + assert len(dataset.test_set) == 1 + + +def test_an_external_tests_root_is_honoured(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)], with_tests=False) + external = tmp_path / "upstream_root" + case_dir = problem_tests_dir(str(external), BATCH_ID) + os.makedirs(case_dir) + dataset = LiveOIBenchDataset(name_or_path=path, tests_root=str(external)) + assert list(dataset.test_set)[0]["tests_dir"] == case_dir + + +def test_a_problem_absent_from_the_test_parquet_is_a_loud_join_failure(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + # A second problem in the problems parquet only -- the two repos out of step. + problems_dir = tmp_path / "LiveOIBench" / "data" + pq.write_table( + pa.Table.from_pylist([_problem(BATCH_ID), _problem(DASHED_ID)]), + problems_dir / "liveoibench_v1.parquet", + ) + with pytest.raises(ValueError, match="out of step"): + LiveOIBenchDataset(name_or_path=path) + + +def test_a_missing_year_parquet_names_the_download_command(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + os.remove(tmp_path / "LiveOIBench_tests" / year_parquet_name("2025")) + with pytest.raises(FileNotFoundError, match="sieval dataset download"): + LiveOIBenchDataset(name_or_path=path) diff --git a/tests/unit/tasks/test_liveoibench_0shot_gen.py b/tests/unit/tasks/test_liveoibench_0shot_gen.py new file mode 100644 index 00000000..d4d6eef8 --- /dev/null +++ b/tests/unit/tasks/test_liveoibench_0shot_gen.py @@ -0,0 +1,443 @@ +"""Unit tests for the LiveOIBench 0-shot chat task. + +Three things are load-bearing here and none of them live in the evaluator: + +* what goes on the wire — the problem's own limits, the grader sources, and a + test *directory* rather than ~140 MB of inlined cases; +* mapping the returned verdict vector onto subtasks by the names the evaluator + reports, rather than by an ordering the task re-derives; +* best-of-n, which is how upstream reports a model — not mean, not pass@k. + +The execution guard itself is the vendored evaluator's, a separate deployable +with its own tests. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import json +import os + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import Request, Response +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + NonRetriableSampleError, + TaskContext, + build_prediction_record, +) +from sieval.datasets.liveoibench import LiveOIBenchDataset +from sieval.tasks.liveoibench_0shot_gen import LiveOIBenchZeroShotGenTask +from tests.conftest import HandlerTransport + +PROBLEM_ID = "IOI-2025-contest-beechtree" +# Two subtasks: the first pays 30 for both its tests, the second 70 for its one. +SUBTASKS = { + "1": {"task": "Subtask 1", "score": 30, "testcases": ["a1", "a2"]}, + "2": {"task": "Subtask 2", "score": 70, "testcases": ["b1"]}, +} +CASE_NAMES = ["a1", "a2", "b1"] + + +class _StubChatModel(ChatModel): + def __init__( + self, replies: tuple[str, ...] = ("```beechtree.cpp\nint main(){}\n```",) + ): + self._replies = replies + super().__init__(model="mock-chat", api_key="fake") + + def _build_default_transport(self) -> HandlerTransport: + return HandlerTransport(self._stub_arun, "openai_chat") + + async def _stub_arun(self, req: Request) -> Response: + return Response(texts=self._replies[: req.sampling.n]) + + +class _Response: + def __init__(self, payload: dict): + self._payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return self._payload + + +class _Evaluator: + """Stands in for the code-eval service, one scripted reply per rollout.""" + + def __init__(self, verdict_sets: list[list[bool]]): + self._verdict_sets = verdict_sets + self.bodies: list[dict] = [] + self.deadlines: list[float] = [] + + async def post(self, url, *, json, timeout): + _ = url + self.bodies.append(json) + self.deadlines.append(timeout) + verdicts = self._verdict_sets[len(self.bodies) - 1] + return _Response( + { + "status": all(verdicts), + "msg": "graded", + "data": { + "n_cases": len(verdicts), + "n_passed": sum(verdicts), + "case_verdicts": verdicts, + "case_names": CASE_NAMES, + "peak_memory_mb": 12.5, + }, + } + ) + + async def aclose(self) -> None: + return None + + +def _raw( + tests_dir: str = "/staged/IOI/2025/contest/beechtree/tests", **overrides +) -> dict: + return { + "problem_id": PROBLEM_ID, + "competition": "IOI", + "contest": "contest", + "task_name": "beechtree", + "problem_statement": "# Beech Tree\n", + "time_limit": 1.5, + "memory_limit": 2048.0, + "task_type": "batch", + "difficulty": 12, + "algorithms": '["tree"]', + "grader_code": "", + "starter_code": "", + "subtasks": json.dumps(SUBTASKS), + "contest_id": "IOI-2025-contest", + "tests_dir": tests_dir, + } | overrides + + +def _task(raw: dict, evaluator: _Evaluator | None = None, **kwargs): + dataset = LiveOIBenchDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([raw])}) + ) + task = LiveOIBenchZeroShotGenTask(dataset, _StubChatModel(), **kwargs) + if evaluator is not None: + task._http_client = evaluator # the real client is never used + return task + + +async def _grade(raw: dict, predictions: list[str], verdict_sets, **kwargs): + evaluator = _Evaluator(verdict_sets) + task = _task(raw, evaluator, **kwargs) + try: + _, judgement = await task.feedback( + build_prediction_record(predictions), + TaskContext(sample_id=0, raw_sample=raw), + ) + finally: + await task.shutdown() + return judgement, evaluator + + +# --------------------------------------------------------------------------- # +# Prompt and extraction +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_the_prompt_is_a_single_user_turn(): + # Upstream's default run sends no system message. + task = _task(_raw()) + record = await task.preprocess(_raw(), TaskContext(sample_id=0)) + await task.shutdown() + assert [m["role"] for m in record["prompt"]] == ["user"] + assert "Time limit: 1.5 seconds" in record["prompt"][0]["content"] + assert "'```beechtree.cpp'" in record["prompt"][0]["content"] + + +@pytest.mark.anyio +async def test_extraction_prefers_the_tasks_own_fence(): + task = _task(_raw()) + reply = ( + "Here is a sketch:\n```cpp\nint wrong(){}\n```\n" + "And the answer:\n```beechtree.cpp\nint right(){}\n```\n" + ) + record = await task.postprocess( + Response(texts=(reply,)), TaskContext(sample_id=0, raw_sample=_raw()) + ) + await task.shutdown() + assert record["rollouts"][0]["prediction"] == "int right(){}" + + +@pytest.mark.anyio +async def test_a_reply_with_no_code_records_a_miss(): + task = _task(_raw()) + record = await task.postprocess( + Response(texts=("I cannot solve this.",)), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + await task.shutdown() + assert record["rollouts"][0].get("prediction") is None + assert record["rollouts"][0]["extracted"] is False + + +# --------------------------------------------------------------------------- # +# What goes on the wire +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_the_request_carries_the_problems_own_limits_and_a_test_directory(): + _, evaluator = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + (body,) = evaluator.bodies + assert body["source"] == "liveoibench" and body["lang"] == "cpp" + assert body["timeout_per_case"] == 1.5 + assert body["memory_limit"] == 2048 + assert body["entry_filename"] == "beechtree.cpp" + # A directory, not ~140 MB of inlined cases. + assert body["test_dir"].endswith("beechtree/tests") + assert "test" not in body + + +@pytest.mark.anyio +async def test_grader_sources_are_sent_for_the_problems_that_ship_them(): + raw = _raw( + grader_code=json.dumps( + {"grader.cpp": "int main(){}", "beechtree.h": "void f();"} + ) + ) + _, evaluator = await _grade(raw, ["void f(){}"], [[True, True, True]]) + assert evaluator.bodies[0]["files"] == { + "grader.cpp": "int main(){}", + "beechtree.h": "void f();", + } + + +@pytest.mark.anyio +async def test_inline_mode_ships_the_cases_read_from_disk(tmp_path): + for name, payload in (("a1", "1\n"), ("a2", "2\n"), ("b1", "3\n")): + (tmp_path / f"{name}.in").write_text(payload) + (tmp_path / f"{name}.out").write_text(payload) + raw = _raw(tests_dir=str(tmp_path)) + _, evaluator = await _grade( + raw, ["int main(){}"], [[True, True, True]], inline_tests=True + ) + body = evaluator.bodies[0] + assert "test_dir" not in body + assert body["test"]["names"] == CASE_NAMES + assert body["test"]["inputs"] == ["1\n", "2\n", "3\n"] + + +@pytest.mark.anyio +async def test_inline_mode_refuses_a_test_input_with_no_expected_output(tmp_path): + (tmp_path / "a1.in").write_text("1\n") + raw = _raw(tests_dir=str(tmp_path)) + with pytest.raises(NonRetriableSampleError, match="no .out"): + await _grade(raw, ["int main(){}"], [[True]], inline_tests=True) + + +@pytest.mark.anyio +async def test_the_http_deadline_outlasts_every_budget_the_evaluator_can_spend( + tmp_path, +): + for name in CASE_NAMES: + (tmp_path / f"{name}.in").write_text("1\n") + (tmp_path / f"{name}.out").write_text("1\n") + _, evaluator = await _grade( + _raw(tests_dir=str(tmp_path)), ["int main(){}"], [[True, True, True]] + ) + # 60s compile bound + 3 cases * (1.5 * 1.2 + 5) + 30s slack. + assert evaluator.deadlines[0] == pytest.approx(60.0 + 3 * 6.8 + 30.0) + + +# --------------------------------------------------------------------------- # +# Verdicts to subtask scores +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_verdicts_are_mapped_onto_subtasks_by_the_names_the_evaluator_reports(): + # b1 alone pays 70; a1/a2 are incomplete so subtask 1 pays nothing. + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, False, True]]) + assert judgement["score"] == 70.0 + assert judgement["metrics"]["relative_score"] == 70.0 + assert judgement["metrics"]["tests_passed_pct"] == pytest.approx(200 / 3) + assert judgement["rollouts"][0]["extra"]["subtasks"]["1"]["score"] == 0 + assert judgement["rollouts"][0]["extra"]["subtasks"]["2"]["score"] == 70 + + +@pytest.mark.anyio +async def test_a_full_solve_is_correct_and_aces(): + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + assert judgement["rollouts"][0]["correct"] is True + assert judgement["metrics"]["ace"] is True + assert judgement["score"] == 100.0 + + +@pytest.mark.anyio +async def test_the_judgement_records_no_reference_and_names_the_rubric(): + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + assert judgement["reference"] is None + assert judgement["extra"]["total_points"] == 100 + assert judgement["extra"]["n_subtasks"] == 2 + assert judgement["extra"]["problem_id"] == PROBLEM_ID + + +@pytest.mark.anyio +async def test_the_best_rollout_is_reported_not_the_first_or_the_mean(): + # Upstream samples n and reports the highest-scoring candidate. + judgement, _ = await _grade( + _raw(), + ["int a(){}", "int b(){}", "int c(){}"], + [[True, True, False], [True, False, True], [False, False, False]], + n=3, + ) + assert judgement["score"] == 70.0, "the 70-point rollout must win" + assert judgement["extra"]["best_rollout"] == 1 + assert judgement["n_rollouts"] == 3 + + +@pytest.mark.anyio +async def test_ties_on_score_are_broken_by_tests_passed(): + # Both score 0; the second passed more tests, so it is the better candidate. + judgement, _ = await _grade( + _raw(), + ["int a(){}", "int b(){}"], + [[False, False, False], [True, False, False]], + n=2, + ) + assert judgement["score"] == 0.0 + assert judgement["extra"]["best_rollout"] == 1 + + +@pytest.mark.anyio +async def test_an_evaluator_without_the_cpp_path_is_an_error_not_a_zero(): + # `status` alone cannot attribute subtasks, so scoring from it would report + # every partial solution as zero. + class _OldEvaluator(_Evaluator): + async def post(self, url, *, json, timeout): + self.bodies.append(json) + return _Response( + {"status": False, "msg": "not supported data source", "data": None} + ) + + task = _task(_raw(), _OldEvaluator([])) + try: + with pytest.raises(RuntimeError, match="no per-case verdicts"): + await task.feedback( + build_prediction_record(["int main(){}"]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_a_problem_with_no_rubric_fails_the_sample_without_retrying(): + task = _task(_raw(subtasks="{}")) + try: + with pytest.raises(NonRetriableSampleError, match="no subtasks"): + await task.feedback( + build_prediction_record(["int main(){}"]), + TaskContext(sample_id=0, raw_sample=_raw(subtasks="{}")), + ) + finally: + await task.shutdown() + + +# --------------------------------------------------------------------------- # +# Report +# --------------------------------------------------------------------------- # +class _Final: + """A finalized sample as `report` reads it: the judgement plus the + prediction record `health_metrics` counts extraction misses from.""" + + def __init__(self, judgement: dict, predictions: list[str | None] | None = None): + self.feedback_result = judgement + self.postprocess_result = build_prediction_record( + predictions if predictions is not None else ["int main(){}"] + ) + + +@pytest.mark.anyio +async def test_the_report_averages_over_every_requested_sample(): + task = _task(_raw()) + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + try: + # One perfect problem plus one pipeline failure -> 50, not 100. + report = await task.report([_Final(judgement)], [object()]) + finally: + await task.shutdown() + assert report["relative_score"] == 50.0 + assert report["score"] == report["relative_score"] + assert report["score_key"] == "relative_score" + assert report["denominator_policy"] == "requested" + assert report["ace_rate"] == 50.0 + assert report["fails"] == 1.0 + + +@pytest.mark.anyio +async def test_an_empty_run_still_declares_its_headline(): + task = _task(_raw()) + try: + report = await task.report([], [object()]) + finally: + await task.shutdown() + assert report["score"] == 0.0 + assert report["score_key"] == "relative_score" + assert report["denominator_policy"] == "requested" + + +@pytest.mark.anyio +async def test_human_metrics_come_from_the_staged_contestant_table(tmp_path): + import pyarrow as pa + import pyarrow.parquet as pq + + contestants = tmp_path / "contest_results.parquet" + pq.write_table( + pa.Table.from_pylist( + [ + { + "contest_id": "IOI-2025-contest", + "gold_cutoff": 90.0, + "silver_cutoff": 60.0, + "bronze_cutoff": 30.0, + "contestants_ranking": json.dumps( + [{"Rank": 1, "beechtree": 100}, {"Rank": 2, "beechtree": 40}] + ), + } + ] + ), + contestants, + ) + + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + task = _task(_raw()) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement)], []) + finally: + await task.shutdown() + + # 100 beats the 40 and ties the 100 -> 50th percentile, and clears gold. + assert report["human_percentile"] == 50.0 + assert report["gold_rate"] == 100.0 + assert report["n_contests"] == 1.0 + + +@pytest.mark.anyio +async def test_a_missing_contestant_table_leaves_the_scores_reportable(tmp_path): + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + task = _task(_raw()) + task.dataset._contestants_path = str(tmp_path / "absent.parquet") + try: + report = await task.report([_Final(judgement)], []) + finally: + await task.shutdown() + assert report["relative_score"] == 100.0 + assert "human_percentile" not in report + + +@pytest.mark.anyio +async def test_the_task_is_registered_under_its_file_name(): + from sieval.core.tasks.meta import get_task_class + + assert get_task_class("liveoibench_0shot_gen") is LiveOIBenchZeroShotGenTask + assert os.path.basename(__file__) == "test_liveoibench_0shot_gen.py" diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index 3b8b8a69..870a5e04 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -481,3 +481,50 @@ Two kinds, and the difference is a decision rather than a status: -H "Accept: application/vnd.oci.image.manifest.v1+json" \ "https://ghcr.io/v2/nuprl/agnostics/manifests/lua" | grep -i docker-content-digest ``` +- `app/exec_cpp.py`, `app/server.py`, `docker/Dockerfile.cpp` — **C++ execution**, + for `source="liveoibench"`. The first non-Python language the evaluator runs as + a *compiled* artifact: `g++ -std=gnu++17 -Wall -O2 -pipe -static -g`, then one + child per test case under `RLIMIT_CPU` / `RLIMIT_AS`. Both limits carry + upstream LiveOIBench's explicit 20% buffer, and a 10 ms poller kills a child + whose CPU time passes the buffered limit — the rule its `BatchJudge` applies, + ported rather than reinvented, along with its output comparison (strip; + single-number outputs compare at `rel_tol=abs_tol=1e-6`; else exact; else line + count plus stripped per-line match). + + Three differences from upstream LiveOIBench, all documented in the module: + compilation is bounded (`timeout`, default 60 s) where upstream's + `subprocess.run` has no timeout; every test always runs, since subtask scoring + needs the whole verdict vector; and no checker path exists, because the + published dataset ships no `checkers/` directory, so upstream's own judge + compares outputs directly on this data. + + `Sample` gains `files` (extra sources compiled alongside — `grader.cpp`, + `{task}.h`) and `entry_filename`; the test model gains `names`, used only in + the failure message. `ResourceMetrics` gains `case_verdicts` — one bool per + case, in request order — because an olympiad subtask scores on its own test + group, so `n_passed` alone cannot be attributed. All fields are optional and + the other sources are untouched. + + The test model is still named `LiveCodeBenchTest` though two sources now share + it; renaming it would widen the diff against upstream without changing the + wire format. + + `Dockerfile.cpp` exists because the base image has no toolchain: g++ without + `libstdc++-*-dev` / `libc6-dev` links nothing statically, and every submission + would fail identically. Verified against g++ 14.2 on Debian: correct, partial, + TLE, MLE, compile-error, float-tolerance and grader-linked submissions all + produce the expected verdict vectors. Not yet upstream — land in + `scitix/code-evaluator` and re-vendor. + + `Dockerfile.multipl-e` also serves this source — it installs `g++`, which + depends on `libstdc++-*-dev` and so pulls `libc6-dev` with it, the same two + packages named explicitly here. An operator running both benchmarks needs one + image, not two; this one stays as the minimal single-purpose build. + + **Not the same C++ path as `exec_lang`'s `cpp` row above**, and the two are not + merge candidates. That one is direct-run: one program, one all-or-nothing + verdict, budgets from the table. This one compiles against the problem's own + grader, runs one child *per official test case* under that problem's + `RLIMIT_CPU` / `RLIMIT_AS`, and returns the whole verdict vector — which is the + only shape IOI subtask scoring can be computed from. They are reached by + different `source` values and share no code. diff --git a/vendor/code-evaluator/app/exec_cpp.py b/vendor/code-evaluator/app/exec_cpp.py new file mode 100644 index 00000000..7239c0c7 --- /dev/null +++ b/vendor/code-evaluator/app/exec_cpp.py @@ -0,0 +1,406 @@ +"""C++ compile-and-run path, for LiveOIBench-style olympiad judging. + +Mirrors the reference judge of +https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/batch_judge.py +(``BatchJudge.run_test_case`` / ``BaseJudge.compile_cpp``), which is the only +place a LiveOIBench verdict is defined: + +* compile with ``g++ -std=gnu++17 -Wall -O2 -pipe -static -g``, the problem's + ``grader.cpp`` first on the command line when it has one; +* per test case, ``RLIMIT_CPU = ceil(time_limit * 1.2)`` and + ``RLIMIT_AS = ceil(memory_limit_mb * 1.2) MB`` -- upstream's explicit 20% + buffer -- with a poller that kills the process once its CPU time passes the + same buffered limit; +* a non-zero exit is a failure; otherwise stdout is compared against the + expected output by ``_compare_outputs`` below. + +Two deliberate differences, both bounds rather than rules: + +* **Compilation is bounded** (``compile_timeout``, default 60 s). Upstream calls + ``subprocess.run`` with no timeout, so a submission whose template expansion + never terminates wedges the judge. Nothing legitimate approaches the bound -- + a solution that cannot compile in a minute at ``-O2`` also cannot be measured. +* **Every test runs.** Upstream's ``stop_on_failure`` mode short-circuits a + subtask at its first failure; here the full verdict vector always comes back, + because subtask scoring needs it and a truncated vector silently scores every + partially-correct submission as zero. Upstream's default is also to run them + all. + +Unlike the LiveCodeBench path, this one executes a *compiled binary* rather than +interpreting a string, so the submission never shares an address space with the +evaluator. +""" + +import asyncio +import math +import os +import re +import resource +import subprocess +import tempfile +import threading +import time +from dataclasses import dataclass + +import psutil +from loguru import logger + +from .resource_monitor import ResourceStats + +COMPILE_COMMAND = ["g++", "-std=gnu++17", "-Wall", "-O2", "-pipe", "-static", "-g"] + +# Upstream's per-test wall clock: `process.communicate(input=..., timeout=120)`. +# The real budget is RLIMIT_CPU; this only catches a process that is not burning +# CPU at all (blocked on a read, sleeping). +WALL_TIMEOUT = 120.0 + +# Upstream's `cpu_time_limit *= 1.2` / `memory_limit_mb *= 1.2`. +LIMIT_BUFFER = 1.2 + +_NUMBER_RE = re.compile(r"^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$") +_LEADING_NUMBER_RE = re.compile(r"^[+\-]?(\d+(\.\d*)?|\.\d+)([eE][+\-]?\d+)?") + + +@dataclass +class CaseResult: + correct: bool + detail: str + cpu_time: float + memory_mb: float + + +@dataclass +class TestCase: + """One test case, either inline or on disk. + + A LiveOIBench problem averages ~140 MB of test data, so a run that shipped + every case inline would move the whole 33.5 GB corpus over HTTP once per + rollout. When the evaluator can see the materialized test tree, the caller + sends the directory instead and each case is read only while it runs. + """ + + name: str + input_data: str | None = None + expected: str | None = None + input_path: str | None = None + expected_path: str | None = None + + def read_input(self) -> str: + if self.input_path is not None: + with open(self.input_path, encoding="utf-8", errors="replace") as f: + return f.read() + return self.input_data or "" + + def read_expected(self) -> str: + if self.expected_path is not None: + with open(self.expected_path, encoding="utf-8", errors="replace") as f: + return f.read() + return self.expected or "" + + +def read_test_dir(test_dir: str) -> list[TestCase]: + """Collect ``{name}.in`` / ``{name}.out`` pairs from a materialized problem. + + Upstream (``Problem.get_test_inputs`` / ``get_test_outputs``) sorts the two + globs independently and pairs them by position, which silently misaligns + every later case if one ``.out`` is missing. Pairing by stem instead makes + that a loud error; on well-formed data the two agree, since sorting by name + *is* sorting by stem. + """ + if not os.path.isdir(test_dir): + raise FileNotFoundError(f"test directory not found: {test_dir}") + + entries = os.listdir(test_dir) + inputs = sorted(name[: -len(".in")] for name in entries if name.endswith(".in")) + outputs = {name[: -len(".out")] for name in entries if name.endswith(".out")} + + missing = [stem for stem in inputs if stem not in outputs] + if missing: + raise FileNotFoundError( + f"{len(missing)} test input(s) in {test_dir} have no matching .out " + f"(e.g. {missing[0]!r})" + ) + return [ + TestCase( + name=stem, + input_path=os.path.join(test_dir, f"{stem}.in"), + expected_path=os.path.join(test_dir, f"{stem}.out"), + ) + for stem in inputs + ] + + +def _is_single_number(text: str) -> bool: + """Upstream ``BatchJudge._is_single_number``.""" + lines = text.splitlines() + while lines and lines[-1].strip() == "": + lines.pop() + if len(lines) != 1: + return False + return bool(_NUMBER_RE.fullmatch(lines[0].strip())) + + +def _compare_numbers(gold: str, output: str) -> bool: + """Upstream ``BatchJudge._compare_numbers``.""" + m_gold = _LEADING_NUMBER_RE.match(gold.strip()) + m_out = _LEADING_NUMBER_RE.match(output.strip()) + if m_gold and m_out: + return math.isclose( + float(m_gold.group(0)), float(m_out.group(0)), rel_tol=1e-6, abs_tol=1e-6 + ) + return False + + +def _compare_outputs(gold_output: str, stdout_decoded: str) -> bool: + """Upstream ``BatchJudge._compare_outputs``. + + Both sides stripped; a single number on both sides compares with tolerance; + then exact match; then line count plus a stripped per-line match. + """ + gold_output = gold_output.strip() + stdout_decoded = stdout_decoded.strip() + + if _is_single_number(gold_output) and _is_single_number(stdout_decoded): + return _compare_numbers(gold_output, stdout_decoded) + + if gold_output == stdout_decoded: + return True + + gold_lines = gold_output.splitlines() + output_lines = stdout_decoded.splitlines() + if len(gold_lines) != len(output_lines): + return False + for gold_line, output_line in zip(gold_lines, output_lines): + if gold_line.strip() != output_line.strip(): + return False + return True + + +def _set_limits(cpu_time_limit: int, memory_limit_bytes: int): + """Upstream ``set_limits``, applied in the child before exec.""" + resource.setrlimit(resource.RLIMIT_CPU, (cpu_time_limit, cpu_time_limit)) + resource.setrlimit(resource.RLIMIT_AS, (memory_limit_bytes, memory_limit_bytes)) + + +def _monitor_process(proc: psutil.Process, time_limit: float, monitor_data: dict, interval=0.01): + """Upstream ``monitor_process``: sample CPU/RSS, kill once CPU passes the limit.""" + while proc.is_running(): + try: + cpu_time = proc.cpu_times().user + proc.cpu_times().system + mem_usage = proc.memory_info().rss / (1024 * 1024) + monitor_data["max_cpu_time"] = max(monitor_data["max_cpu_time"], cpu_time) + monitor_data["max_memory"] = max(monitor_data["max_memory"], mem_usage) + if cpu_time > time_limit: + proc.kill() + break + except psutil.NoSuchProcess: + break + time.sleep(interval) + + +def _compile( + workdir: str, + entry_filename: str, + grader_filename: str | None, + compile_timeout: float, +) -> tuple[bool, str]: + """Upstream ``BaseJudge.compile_cpp``, run inside the submission's workdir. + + ``cwd=workdir`` is what makes ``#include "task.h"`` resolve, the same way the + upstream judge's copied-together working directory does. + """ + sources = [grader_filename, entry_filename] if grader_filename else [entry_filename] + command = [*COMPILE_COMMAND, "-o", "solution", *sources] + try: + result = subprocess.run( + command, + cwd=workdir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=compile_timeout, + check=True, + ) + except subprocess.CalledProcessError as e: + return False, e.stderr or e.stdout or "compilation failed" + except subprocess.TimeoutExpired: + return False, f"compilation timed out after {compile_timeout}s" + except FileNotFoundError: + return False, "g++ not found: this evaluator image has no C++ toolchain" + return True, result.stdout + + +def _run_one_test( + workdir: str, + case: TestCase, + cpu_time_limit: float, + memory_limit_mb: float, +) -> CaseResult: + """One test case, with upstream's buffered limits and comparison.""" + input_data = case.read_input() + # Upstream buffers both limits by 20% before applying them. + cpu_time_limit = cpu_time_limit * LIMIT_BUFFER + memory_limit_mb = memory_limit_mb * LIMIT_BUFFER + + executable = os.path.join(workdir, "solution") + + def preexec(): + _set_limits( + int(math.ceil(cpu_time_limit)), + int(math.ceil(memory_limit_mb * 1024 * 1024)), + ) + + process = subprocess.Popen( + [executable], + cwd=workdir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=preexec, + ) + + monitor_data = {"max_cpu_time": 0.0, "max_memory": 0.0} + monitor_thread = None + try: + proc = psutil.Process(process.pid) + monitor_thread = threading.Thread( + target=_monitor_process, args=(proc, cpu_time_limit, monitor_data), daemon=True + ) + monitor_thread.start() + except psutil.NoSuchProcess: + # Exited before we could attach; the limits in the child still applied. + pass + + try: + stdout_data, stderr_data = process.communicate( + input=input_data.encode(), timeout=WALL_TIMEOUT + ) + except subprocess.TimeoutExpired: + process.kill() + stdout_data, stderr_data = process.communicate() + if monitor_thread is not None: + monitor_thread.join() + + cpu_time = monitor_data["max_cpu_time"] + memory_mb = monitor_data["max_memory"] + + if process.returncode == 0: + stdout_decoded = (stdout_data.decode(errors="replace") if stdout_data else "").strip() + correct = _compare_outputs(case.read_expected(), stdout_decoded) + return CaseResult(correct, "" if correct else "wrong answer", cpu_time, memory_mb) + + # A non-zero exit is a failure regardless of cause; the cause is reported + # only as free text, because upstream's judge does not classify it either -- + # `ResultType` exists in its tree but `BatchJudge` never assigns one. + if cpu_time >= cpu_time_limit: + detail = f"time limit exceeded ({cpu_time:.2f}s of {cpu_time_limit:.2f}s)" + elif process.returncode == -9: + detail = f"killed (peak rss {memory_mb:.0f}MB of {memory_limit_mb:.0f}MB)" + else: + stderr_text = (stderr_data.decode(errors="replace") if stderr_data else "").strip() + detail = f"runtime error [exit {process.returncode}]: {stderr_text[:200]}" + return CaseResult(False, detail, cpu_time, memory_mb) + + +def build_cases( + inputs: list[str] | None, + expect_outputs: list[str] | None, + names: list[str] | None = None, + test_dir: str | None = None, +) -> list[TestCase]: + """Normalize the two ways a caller can supply tests into one case list.""" + if test_dir: + return read_test_dir(test_dir) + inputs = inputs or [] + expect_outputs = expect_outputs or [] + return [ + TestCase( + name=names[i] if names and i < len(names) else f"#{i}", + input_data=input_data, + expected=expected, + ) + for i, (input_data, expected) in enumerate(zip(inputs, expect_outputs)) + ] + + +async def execute_tests( + code: str, + cases: list[TestCase], + entry_filename: str = "solution.cpp", + files: dict[str, str] | None = None, + timeout_per_case: float = 1.0, + memory_limit: int = 1024, + compile_timeout: float = 60.0, + max_workers: int = 4, +) -> tuple[bool, str, ResourceStats, int, list[bool]]: + """Compile *code* and run it against every case. + + Returns ``(all_passed, msg, stats, n_passed, verdicts)``. ``verdicts`` is one + bool per test **in case order** -- the caller maps them onto subtasks by + index, so the list is always as long as *cases*, including on a compilation + error (all ``False``). + + *files* are extra sources written beside the submission: the problem's + ``grader.cpp`` and ``{task}.h``. A ``grader.cpp`` among them is compiled with + the submission, which is how upstream grades the 41 problems that ship one. + + *max_workers* only bounds host load: verdicts are decided by each child's own + CPU-time and address-space limits, so they do not depend on how many run at + once. + """ + files = dict(files or {}) + stats = ResourceStats() + + with tempfile.TemporaryDirectory(prefix="liveoibench-") as workdir: + for filename, contents in files.items(): + # Flat by construction: upstream's bundles are `grader.cpp` / `task.h`. + with open(os.path.join(workdir, os.path.basename(filename)), "w") as f: + f.write(contents) + with open(os.path.join(workdir, entry_filename), "w") as f: + f.write(code) + + grader_filename = "grader.cpp" if "grader.cpp" in files else None + compiled, compile_msg = await asyncio.to_thread( + _compile, workdir, entry_filename, grader_filename, compile_timeout + ) + if not compiled: + logger.debug(f"compilation failed:\n{compile_msg}") + return ( + False, + f"compilation error: {compile_msg.strip()[:500]}", + stats, + 0, + [False] * len(cases), + ) + + semaphore = asyncio.Semaphore(max(1, max_workers)) + + async def run_case(case: TestCase) -> CaseResult: + async with semaphore: + return await asyncio.to_thread( + _run_one_test, workdir, case, timeout_per_case, memory_limit + ) + + results: list[CaseResult] = list( + await asyncio.gather(*(run_case(case) for case in cases)) + ) + + verdicts = [result.correct for result in results] + n_passed = sum(verdicts) + all_passed = n_passed == len(verdicts) and bool(verdicts) + + if results: + stats.peak_memory_mb = max(result.memory_mb for result in results) + stats.memory_mb = sum(result.memory_mb for result in results) / len(results) + + if all_passed: + msg = f"passed {n_passed}/{len(verdicts)} test cases" + else: + first_failure = next(i for i, result in enumerate(results) if not result.correct) + msg = ( + f"passed {n_passed}/{len(verdicts)} test cases; " + f"first failure at test {cases[first_failure].name}: " + f"{results[first_failure].detail}" + ) + + return all_passed, msg, stats, n_passed, verdicts diff --git a/vendor/code-evaluator/app/server.py b/vendor/code-evaluator/app/server.py index 2ff0b866..9f07a72a 100644 --- a/vendor/code-evaluator/app/server.py +++ b/vendor/code-evaluator/app/server.py @@ -10,6 +10,8 @@ from pydantic import BaseModel from .exec_agnostics import execute_agnostics +from .exec_cpp import build_cases +from .exec_cpp import execute_tests as exec_cpp_test from .exec_js import execute_code as exec_js from .exec_lang import LANGUAGES, toolchain_entry, toolchain_present from .exec_lang import execute_code as exec_lang @@ -74,6 +76,18 @@ class ResourceMetrics(BaseModel): ``None`` only when nothing ran at all, i.e. an unsupported language or source, where ``data`` itself is ``None``. + ``case_verdicts`` is the per-case detail behind ``n_passed``: one bool per + case, in request order. Only ``liveoibench`` fills it, because only it needs + *which* cases passed -- an olympiad subtask scores on its own test group, so + a count cannot be attributed. ``None`` elsewhere means "not reported", which + is why the count fields stay rather than being derived from it. + + ``case_names`` is what those verdicts belong to, positionally. It matters + when the caller passed ``test_dir`` rather than inline cases: the server + decided the order by listing that directory, and a caller that re-derived it + would be keeping a second copy of the ordering rule -- the kind that agrees + until one side changes. + Kept as one flat model rather than a per-source subclass: FastAPI filters the response against this route's declared model, so a subclass returned from one branch would have its extra fields silently stripped. @@ -85,6 +99,8 @@ class ResourceMetrics(BaseModel): peak_memory_mb: float n_cases: int | None = None n_passed: int | None = None + case_verdicts: list[bool] | None = None + case_names: list[str] | None = None # quotebench only. They live on this flat model for the reason the docstring # above gives: a per-source subclass would have these stripped by FastAPI's # response filtering. `error_class` is the failure taxonomy @@ -270,6 +286,10 @@ class LiveCodeBenchTest(BaseModel): inputs: list[str] outputs: list[str] fn_name: str | None = None + # `liveoibench` only, and only for the failure message: its test cases have + # official names ("boi1a") that a caller reading the log wants back. The + # verdict-to-subtask mapping is positional, so nothing depends on it. + names: list[str] | None = None class Sample(BaseModel): @@ -279,6 +299,16 @@ class Sample(BaseModel): test: LiveCodeBenchTest | None = None lang: str = "python" timeout: float | None = None + # `liveoibench` only: extra sources written beside the submission before + # compiling -- the problem's `grader.cpp` and `{task}.h` -- and the filename + # the submission itself is written as, which its `#include` has to match. + files: dict[str, str] | None = None + entry_filename: str | None = None + # `liveoibench` only: a directory of `{name}.in` / `{name}.out` pairs this + # evaluator can read, used instead of `test`. A problem averages ~140 MB of + # test data, so inlining it would move the corpus over HTTP once per rollout; + # a caller whose evaluator cannot see the volume sends `test` instead. + test_dir: str | None = None # Per-case budget, applied on top of `timeout`, which stays a whole-suite wall. # Opt-in: absent keeps the previous behaviour exactly. Official LiveCodeBench # budgets per case rather than per suite -- see `exec_py_test.execute_test`. @@ -544,6 +574,96 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: verifier_image=verifier_image, ), ) + elif sample.source == "liveoibench": + # 'liveoibench': compile the C++ submission once, then run every official + # test case. Unlike the livecodebench path this reports per-case verdicts + # and never stops early -- an olympiad subtask is scored on its own test + # group, so a submission that fails case 0 and passes the rest still + # earns points. + if sample.lang != "cpp": + return BasicResponse( + status=False, msg=f"not supported language: {sample.lang}", data=None + ) + if sample.test is None and not sample.test_dir: + return BasicResponse( + status=False, + msg="'liveoibench' requires either 'test' or 'test_dir'", + data=None, + ) + if ( + sample.test is not None + and not sample.test_dir + and len(sample.test.inputs) != len(sample.test.outputs) + ): + return BasicResponse( + status=False, + msg=( + f"test inputs/outputs length mismatch: " + f"{len(sample.test.inputs)} != {len(sample.test.outputs)}" + ), + data=None, + ) + + try: + cases = build_cases( + inputs=sample.test.inputs if sample.test else None, + expect_outputs=sample.test.outputs if sample.test else None, + names=sample.test.names if sample.test else None, + test_dir=sample.test_dir, + ) + except (FileNotFoundError, OSError) as e: + # An unreadable `test_dir` is a deployment problem (the evaluator + # cannot see the test volume), not a verdict on the submission. + logger.error(f"cannot read tests for sample '{sample.uuid}': {e}") + return BasicResponse(status=False, msg=f"cannot read tests: {e}", data=None) + + # The problem's own time limit, in seconds, per case; upstream buffers it + # by 20% inside the runner. `timeout` bounds compilation instead -- the + # one step with no per-case budget. + timeout_per_case = ( + sample.timeout_per_case if sample.timeout_per_case is not None else 1.0 + ) + compile_timeout = sample.timeout if sample.timeout is not None else 60.0 + + ok, msg, stats, n_passed, verdicts = await exec_cpp_test( + code=sample.code, + cases=cases, + entry_filename=sample.entry_filename or "solution.cpp", + files=sample.files, + timeout_per_case=timeout_per_case, + memory_limit=sample.memory_limit, + compile_timeout=compile_timeout, + ) + n_cases = len(cases) + + logger.info( + f"evaluate sample '{sample.uuid}' from '{sample.source}', " + f"language: {sample.lang}, timeout_per_case: {timeout_per_case}, " + f"compile_timeout: {compile_timeout}, memory_limit: {sample.memory_limit}, " + f"kwargs: {sample.kwargs}, status: {ok}, msg: {msg}, " + f"cases: {n_passed}/{n_cases}, " + f"avg_memory: {stats.memory_mb:.2f}MB, " + f"peak_memory: {stats.peak_memory_mb:.2f}MB" + ) + return BasicResponse( + status=ok, + msg=msg, + data=ResourceMetrics( + # Both CPU fields are 0.0 on this path and that is measured, not + # missing: the per-case poller samples each child's CPU *time* in + # seconds to enforce RLIMIT_CPU, and there is no meaningful way to + # render that as the whole-service percentage this field means. + # Per-case CPU seconds stay inside the runner's verdict detail. + avg_cpu_percent=stats.cpu_percent, + peak_cpu_percent=stats.peak_cpu_percent, + avg_memory_mb=stats.memory_mb, + peak_memory_mb=stats.peak_memory_mb, + n_cases=n_cases, + n_passed=n_passed, + case_verdicts=verdicts, + case_names=[case.name for case in cases], + ), + ) else: logger.error(f"not supported data source: {sample.source}") return BasicResponse( diff --git a/vendor/code-evaluator/docker/Dockerfile.cpp b/vendor/code-evaluator/docker/Dockerfile.cpp new file mode 100644 index 00000000..5e52fc57 --- /dev/null +++ b/vendor/code-evaluator/docker/Dockerfile.cpp @@ -0,0 +1,20 @@ +# Image for the `liveoibench` source: the base image has no C++ toolchain, and +# `exec_cpp` shells out to `g++`. `libstdc++-*-dev` and `libc6-dev` are what the +# upstream judge's `-static` link needs; without them g++ is present but every +# submission fails to link, which reads as a benchmark of nothing. +FROM python:3.10-slim + +WORKDIR /opt/code-evaluator + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + libc6-dev \ + libstdc++-12-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ ./app/ + +CMD ["fastapi", "run", "app/server.py", "--port", "11451"] From 62517eb02f74576501ad1275a36842218dcf25b4 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 14:57:07 +0800 Subject: [PATCH 2/5] fix(tasks): align LiveOIBench's human ranking and report with upstream Review of the benchmark found four divergences in the ranking and aggregation half, all of which move numbers the paper can be compared against, and two of which were documented as not existing. * **USACO never joined the contestant table.** The contest id was derived as `{competition}-{year}-{round}`, but the published table keys USACO by division (`USACO-2023-December_Contest-platinum` / `-combined`). Of the 61 contest ids the 380 batch problems derive, 11 were absent from the 72-row table -- 132 problems, 35% of the scored set, silently dropped from `human_percentile`, `medal_rate` and `gold_rate` behind a count-only warning. Upstream's routing is ported instead: `build_problem_to_contest_map` reads the authoritative mapping out of the contestant row's own `problems` column (where the division split is the only place it is stated), and `resolve_contest_id` walks upstream's fallback ladder, CCO rename included. All 380 batch problems are listed by exactly one contest row, so the join is now total. * **`Recalculated_Total` is in the published data.** The module claimed no contest carried it; 10 of 72 do, and upstream prefers it over the per-task sum wherever it exists. The two differ for 20-83% of contestants on each of those contests, and the resulting percentiles differ on four: at a model total of a quarter of the maximum, CEOI-2023 read 75.0 where upstream reads 37.5. The branch is ported, dropping unparseable rows as upstream's `.dropna()` does rather than zeroing them, along with the Canadian Computing Olympiad rank re-derivation nested inside it (unreachable on this release, ported for the same reason the `min-score` branch is). * **`relative_score` was problem-weighted**, where upstream means inside a contest and then across contests. Measured over the 380 scored problems: 72 contests sized 1 to 10, the largest being JOI-2023-JOI_spring at 10, then eleven USACO rounds at 9 each -- so the two weightings are not interchangeable. This is the task's `score_key`. * **`pass_rate` meant the opposite of upstream's.** What was reported as `pass_rate` is upstream's `tests_passed_pct` (fraction of test cases) and what was reported as `ace_rate` is upstream's `pass_rate` (fraction of problems fully solved). Both now carry upstream's names, so the report can be read against `{model}_contest.json` without a translation table. Two contests-with-no-contestants paths came out of the same pass: a contest that publishes cutoffs and an empty ranking still earns its medal (upstream's `df.empty` branch -- every USACO row in the release is this shape), while a USACO `-combined` contest reports neither medal nor percentile, because upstream scores it from promotion thresholds that live in its repository rather than in the dataset. NaN cutoff cells now read as "no cutoff" instead of as a cutoff nothing can clear. Robustness, from the same review: * The evaluator raised `RuntimeError: coroutine raised StopIteration` on a suite with no test case -- it looked for a first failure among no results. It now refuses the suite by name, and the task turns an empty verdict vector into a `NonRetriableSampleError` that names the directory instead of scoring every subtask at zero. * Materialization was not atomic. An interrupted 33.5 GB unpack left a partial directory that both the script's own skip check and the loader's `require_tests` guard read as complete. Cases now land in a sibling `tests.partial` and are renamed over the target in one step. * The per-request HTTP deadline was a bare float, which httpx also applies to the wait for a free connection. These deadlines span 150s to half an hour, so a small problem queued behind large ones could fail on the pool without being graded; `pool=None` decouples them. * The test directory is read once per sample from a worker thread rather than once per rollout on the event loop. * `pyarrow` is declared: it was imported directly by the dataset and only reached the environment as a transitive of `datasets`. Lock re-resolved with `--update-reuse`; the only change is the content hash. `reference_impl.notes` gains the aggregation rule, the USACO ranking situation, and the three IATI problems that link against a grader whose header the prompt never shows -- an unavoidable 0 that upstream shares and that would otherwise read as model failure. Verified: 5875 tests pass (18 new), ruff and ty clean, preflight 25/25 with no warnings. Each fix was reverse-mutated at its new site and every one of the 11 mutants was caught. The C++ path was re-checked against g++ 14.2 for the empty, partial and compile-error cases. Contest and problem counts above were measured against the pinned dataset revisions. Co-Authored-By: Claude Opus 5 (1M context) --- pdm.lock | 2 +- pyproject.toml | 4 + scripts/materialize_liveoibench_tests.py | 38 ++- sieval/community/liveoibench/rankings.py | 252 ++++++++++++++-- sieval/meta/index.json | 2 +- sieval/tasks/liveoibench_0shot_gen.py | 283 +++++++++++++----- .../community/liveoibench/test_rankings.py | 163 ++++++++++ tests/unit/datasets/test_liveoibench.py | 12 +- .../test_materialize_liveoibench_tests.py | 93 ++++++ .../unit/tasks/test_liveoibench_0shot_gen.py | 242 +++++++++++++-- vendor/code-evaluator/VENDORED.md | 13 +- vendor/code-evaluator/app/exec_cpp.py | 11 +- 12 files changed, 977 insertions(+), 138 deletions(-) create mode 100644 tests/unit/scripts/test_materialize_liveoibench_tests.py diff --git a/pdm.lock b/pdm.lock index 9fb06e8a..dd695add 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "nl2sh-alfa", "ruler", "ruler-gen", "scicode", "spider", "spider2", "t-eval", "test", "wikisql"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:ee11aa0c7ca350d44ac483278e81a61f330b54a9b0d1119250001b418fac2e61" +content_hash = "sha256:2733395ff19bb311187e466cf75746088e4f331191d869ff9dedcbe5fc05bc33" [[metadata.targets]] requires_python = ">=3.12,<3.15" diff --git a/pyproject.toml b/pyproject.toml index 2d183ae6..2a2995eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,10 @@ dependencies = [ "openai>=2.6.0", "orjson>=3.11.4", "packaging>=21.0", + # Direct, not just via datasets: sieval/datasets/liveoibench.py reads the + # LiveOIBench parquets with pyarrow, projecting columns so a 33.5 GB `tests` + # column stays unread. Floor matches the one datasets 4.2 already imposes. + "pyarrow>=21.0.0", "pyyaml>=6.0.3", "tqdm>=4.67.1", "typer>=0.24.1", diff --git a/scripts/materialize_liveoibench_tests.py b/scripts/materialize_liveoibench_tests.py index af9b9714..a398e143 100644 --- a/scripts/materialize_liveoibench_tests.py +++ b/scripts/materialize_liveoibench_tests.py @@ -23,6 +23,7 @@ import argparse import json import os +import shutil import sys from pathlib import Path @@ -40,6 +41,33 @@ ) +def write_problem_tests(out_dir: Path, tests: dict) -> None: + """Write one problem's cases so the directory only ever exists complete. + + A 33.5 GB unpack is long enough to be interrupted — Ctrl-C, a full disk, an + OOM kill — and both the skip check above and the dataset loader's + ``require_tests`` guard read a directory's *existence* as "this problem is + materialized". Writing in place would make a half-written problem + indistinguishable from a whole one, and it would surface much later as a + subtask naming a test case that is not there. + + So the cases land in a sibling ``tests.partial`` and are renamed over the + target in one step. A leftover ``.partial`` is a crash marker, and is + discarded the next time this problem is written. + """ + staging = out_dir.with_name(out_dir.name + ".partial") + if staging.is_dir(): + shutil.rmtree(staging) + staging.mkdir(parents=True, exist_ok=True) + for name, case in tests.items(): + case = case or {} + (staging / f"{name}.in").write_text(case.get("input", ""), encoding="utf-8") + (staging / f"{name}.out").write_text(case.get("output", ""), encoding="utf-8") + if out_dir.is_dir(): + shutil.rmtree(out_dir) + os.replace(staging, out_dir) + + def materialize_year( parquet_path: Path, tests_root: Path, overwrite: bool, verbose: bool = True ) -> tuple[int, int]: @@ -63,15 +91,7 @@ def materialize_year( tests = json.loads(payload or "{}") if not isinstance(tests, dict): raise ValueError(f"Malformed tests payload for {problem_id!r}") - out_dir.mkdir(parents=True, exist_ok=True) - for name, case in tests.items(): - case = case or {} - (out_dir / f"{name}.in").write_text( - case.get("input", ""), encoding="utf-8" - ) - (out_dir / f"{name}.out").write_text( - case.get("output", ""), encoding="utf-8" - ) + write_problem_tests(out_dir, tests) written += 1 if verbose: print(f" {problem_id}: {len(tests)} cases") diff --git a/sieval/community/liveoibench/rankings.py b/sieval/community/liveoibench/rankings.py index c79f4810..bd909d49 100644 --- a/sieval/community/liveoibench/rankings.py +++ b/sieval/community/liveoibench/rankings.py @@ -1,6 +1,7 @@ # adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/generate_rankings.py -# (normalize_name, NON_TASK_COLUMNS, identify_task_columns, calculate_percentile, -# and the per-contest branch of compute_human_metrics) +# (normalize_name, normalize_contest_identifier, NON_TASK_COLUMNS, +# identify_task_columns, calculate_percentile, build_problem_to_contest_map, +# the contest-routing block of group_problems_by_contest, and compute_human_metrics) """Ranking a model against the contest's human contestants. Divergence from upstream: @@ -9,13 +10,20 @@ here the rows arrive as the list of dicts that column actually holds, so ``df.columns`` becomes the union of row keys and the vectorized column sum becomes a per-row sum. The numbers are the same. -* Upstream prefers a ``Recalculated_Total`` column when one exists and, on that - branch only, re-derives the Canadian Computing Olympiad percentile from - ``Rank``. No published contest carries that column, so neither branch is - ported; ``score_contest`` implements the column-matching branch that the - released data takes. * Codeforces Elo is not ported — it is a separate upstream fit against contestant CF ratings. +* Upstream's USACO ``-combined`` contests are scored from promotion thresholds + in ``data/USACO/{year}/{contest}/contest_info.json``, a file that lives in the + upstream repository rather than in the published dataset. Without it upstream + itself returns no medal and no percentile for those contests, which is what + :func:`score_contest` returns; see the ``-combined`` branch there. + +Contest identity is **not** derivable from the problem id alone. USACO keys its +contestant rows by division (``USACO-2023-December_Contest-platinum`` / +``-combined``), so :func:`build_problem_to_contest_map` reads the authoritative +mapping out of the contestant table's own ``problems`` column and +:func:`resolve_contest_id` falls back to upstream's division ladder. On the +published data every problem is listed by exactly one contest row. Human totals are re-summed over **only** the tasks the model was scored on, so a contest whose problems were partly filtered out (interactive ones are) still @@ -23,8 +31,9 @@ not rescaled, which is upstream's behaviour. """ +import json import re -from typing import Any, Mapping, Sequence +from typing import Any, Iterable, Mapping, Sequence NON_TASK_COLUMNS = { "rank", @@ -45,6 +54,16 @@ } _FALLBACK_TOTAL_COLUMNS = ("Total", "total", "Total Score", "score", "Score") +_RECALC_TOTAL_COLUMNS = ("Recalculated_Total", "recalculated_total") +_RANK_COLUMNS = ("Rank", "rank") +_PROMOTED_DIVISIONS = {"bronze", "silver", "gold"} + +# Upstream renames the three Canadian rounds when matching the human leaderboard. +_CCO_ROUND_ALIASES = { + "Canadian_Computing_Competition_Junior": "Junior", + "Canadian_Computing_Competition_Senior": "Senior", + "Canadian_Computing_Olympiad": "contest", +} def normalize_name(value: str) -> str: @@ -52,6 +71,115 @@ def normalize_name(value: str) -> str: return re.sub(r"[^a-z0-9]", "", value.lower()) +def normalize_contest_identifier(contest_base: str) -> str: + """Upstream ``normalize_contest_identifier``: rename the CCO rounds. + + The published contestant table happens to key CCO by its *un*-renamed round, + which is why :func:`resolve_contest_id` keeps upstream's fallback to the raw + identifier. Ported so the ladder is the same one upstream walks. + """ + if not contest_base.startswith("CCO-"): + return contest_base + parts = contest_base.split("-", 2) + if len(parts) != 3: + return contest_base + prefix, year, rest = parts + for source, replacement in _CCO_ROUND_ALIASES.items(): + if rest.startswith(source): + rest = rest.replace(source, replacement, 1) + break + return f"{prefix}-{year}-{rest}" + + +def usaco_division(problem_id: str) -> str | None: + """The division token a USACO problem id carries, or ``None`` elsewhere. + + ``USACO-2023-US_Open_Contest-bronze_FEB`` -> ``"bronze"``. Upstream reads it + off the same fourth ``-`` segment, which is also the ``task_name`` column. + """ + if not problem_id.startswith("USACO-"): + return None + parts = problem_id.split("-", 3) + if len(parts) < 4: + return None + return parts[3].split("_", 1)[0].lower() + + +def build_problem_to_contest_map( + contest_rows: Iterable[Mapping[str, Any]], +) -> dict[str, str]: + """Upstream ``build_problem_to_contest_map``: problem id -> contest id. + + Driven by each contestant row's ``problems`` column, which is the only place + the USACO division split is stated. The division filter is upstream's and is + a no-op on consistent data — a ``-combined`` row lists only bronze/silver/gold + problems and a ``-platinum`` row only platinum ones. + """ + mapping: dict[str, str] = {} + for row in contest_rows: + contest_id = row.get("contest_id") + problems = row.get("problems") + if not contest_id or problems is None: + continue + if isinstance(problems, str): + try: + problems = json.loads(problems) + except json.JSONDecodeError: + continue + if not isinstance(problems, (list, tuple)): + continue + for problem in problems: + problem_id = str(problem) + division = usaco_division(problem_id) + if division is None: + mapping[problem_id] = contest_id + elif contest_id.endswith("-combined"): + if division in _PROMOTED_DIVISIONS: + mapping[problem_id] = contest_id + elif contest_id.endswith("-platinum"): + if division == "platinum": + mapping[problem_id] = contest_id + else: + mapping[problem_id] = contest_id + return mapping + + +def resolve_contest_id( + problem_id: str, + contest_base: str, + problem_to_contest: Mapping[str, str], + available_contests: Sequence[str] | set[str] = (), +) -> str | None: + """Which contest row one problem is ranked against; upstream's ladder. + + *contest_base* is ``{competition}-{year}-{round}``, the identifier the + problem id alone yields. ``None`` means the problem has no human ranking — + upstream's ``continue``, which drops a USACO problem whose division token is + not one of the four. + """ + available = set(available_contests) + contest_id = problem_to_contest.get(problem_id) + base_norm = normalize_contest_identifier(contest_base) + + if not contest_id: + division = usaco_division(problem_id) + if division is not None: + if division == "platinum": + contest_id = f"{base_norm}-platinum" + elif division in _PROMOTED_DIVISIONS: + contest_id = f"{base_norm}-combined" + else: + return None + else: + contest_id = base_norm + + if contest_id not in available and contest_id != base_norm and base_norm in available: + contest_id = base_norm + if contest_id not in available and contest_base in available: + contest_id = contest_base + return contest_id + + def identify_task_columns(columns: Sequence[str], contest_id: str = "") -> dict[str, str]: """Return a mapping from normalized task name to original column name.""" mapping: dict[str, str] = {} @@ -107,6 +235,38 @@ def _to_numeric(value: Any) -> float: return 0.0 if number != number else number # NaN -> 0.0 +def _numeric_or_none(value: Any) -> float | None: + """``pd.to_numeric(errors="coerce")`` for a single cell, keeping NaN as None. + + The recalculated-total branch *drops* unparseable rows rather than zeroing + them (upstream ``.dropna()``), so it needs the distinction ``_to_numeric`` + throws away. + """ + if value is None or isinstance(value, bool): + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return None if number != number else number + + +def _result( + human_percentile: float | None, + medal: str | None, + model_total: float, + matched_columns: list[str], + n_contestants: int, +) -> dict: + return { + "human_percentile": human_percentile, + "medal": medal, + "model_total": model_total, + "matched_columns": matched_columns, + "n_contestants": n_contestants, + } + + def score_contest( rankings: Sequence[Mapping[str, Any]], model_scores: Mapping[str, float], @@ -120,11 +280,63 @@ def score_contest( *model_scores* maps a problem's ``task_name`` to the model's score on it, for the problems of this contest that were actually evaluated. """ + model_sum = float(sum(model_scores.values())) + + # Upstream short-circuits a USACO `-combined` contest into + # `compute_usaco_combined_metrics`, which compares each division's total + # against the promotion thresholds in that contest's `contest_info.json`. + # That file is part of the upstream repository, not of the published + # dataset, so upstream on this data reaches `thresholds_present = False` and + # returns no medal and no percentile. Returning anything here -- the row's + # own medal cutoffs, say -- would be a number upstream does not compute. + if contest_id.startswith("USACO-") and contest_id.endswith("-combined"): + return _result(None, None, model_sum, [], len(rankings)) + + # No contestant rows: upstream's `if df.empty` branch still awards a medal + # from the published cutoffs, and only the percentile is unavailable. Every + # USACO contest in the release takes this path. + if not rankings: + medal = medal_from_cutoffs(model_sum, gold_cutoff, silver_cutoff, bronze_cutoff) + return _result(None, medal, model_sum, [], 0) + columns: list[str] = [] for row in rankings: for key in row: if key not in columns: columns.append(key) + + # Upstream prefers a `Recalculated_Total` column wherever one exists: the + # contest's own total re-derived over the tasks LiveOIBench scores, which is + # not the same number as the sum of the per-task columns (it differs for + # 20-83% of contestants on the 10 contests that carry it). The model side + # stays the sum of every problem evaluated in the contest. + recalc_column = next((col for col in _RECALC_TOTAL_COLUMNS if col in columns), None) + if recalc_column is not None: + human_totals = [ + value + for value in (_numeric_or_none(row.get(recalc_column)) for row in rankings) + if value is not None + ] + human_percentile = calculate_percentile(model_sum, human_totals) + # On this branch only, upstream re-derives the Canadian Computing + # Olympiad percentile from the rank column. No published CCO contest + # carries `Recalculated_Total`, so this is unreachable on the released + # data; ported because dropping it would make the port wrong for a + # release that does. + if "canadian_computing_olympiad" in contest_id.lower() and any( + col in columns for col in _RANK_COLUMNS + ): + rank_column = next(col for col in _RANK_COLUMNS if col in columns) + n_ranked = sum( + 1 for row in rankings if _numeric_or_none(row.get(rank_column)) is not None + ) + if n_ranked: + model_rank = sum(1 for total in human_totals if total > model_sum) + 1 + model_rank = max(1, min(model_rank, n_ranked)) + human_percentile = ((n_ranked - model_rank) / n_ranked) * 100 + medal = medal_from_cutoffs(model_sum, gold_cutoff, silver_cutoff, bronze_cutoff) + return _result(human_percentile, medal, model_sum, [recalc_column], len(rankings)) + task_column_map = identify_task_columns(columns, contest_id) matched_columns: list[str] = [] @@ -142,24 +354,18 @@ def score_contest( # model's own total is every evaluated problem summed. fallback = next((col for col in _FALLBACK_TOTAL_COLUMNS if col in columns), None) if fallback is None: - return { - "human_percentile": None, - "medal": None, - "model_total": float(sum(model_scores.values())), - "matched_columns": [], - "n_contestants": len(rankings), - } + return _result(None, None, model_sum, [], len(rankings)) matched_columns = [fallback] - model_total = float(sum(model_scores.values())) + model_total = model_sum human_totals = [ sum(_to_numeric(row.get(column)) for column in matched_columns) for row in rankings ] - return { - "human_percentile": calculate_percentile(model_total, human_totals), - "medal": medal_from_cutoffs(model_total, gold_cutoff, silver_cutoff, bronze_cutoff), - "model_total": model_total, - "matched_columns": matched_columns, - "n_contestants": len(rankings), - } + return _result( + calculate_percentile(model_total, human_totals), + medal_from_cutoffs(model_total, gold_cutoff, silver_cutoff, bronze_cutoff), + model_total, + matched_columns, + len(rankings), + ) diff --git a/sieval/meta/index.json b/sieval/meta/index.json index e412f5e3..7d470495 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -2410,7 +2410,7 @@ "reference_impl": { "source": "liveoibench", "url": "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/base_judge.py", - "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Codeforces Elo is not computed." + "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Aggregation follows upstream's overall block: relative_score is meaned within a contest and then across contests, pass_rate is the fraction of problems fully solved, and tests_passed_pct the fraction of test cases. Human ranking resolves a contest through the contestant table's own 'problems' column, which is where USACO's division split (-platinum / -combined) lives; all 22 USACO rows publish medal cutoffs but no contestant list, so they yield a medal and no percentile, and -combined yields neither -- upstream scores it from promotion thresholds the published dataset does not carry. Three IATI problems (senior-game, senior-ones, junior-twins) link against a grader whose header the prompt never shows, so they cannot compile and score 0; upstream behaves identically. Codeforces Elo is not computed." }, "status": "stable", "reference_kind": "procedure" diff --git a/sieval/tasks/liveoibench_0shot_gen.py b/sieval/tasks/liveoibench_0shot_gen.py index 84362906..54f83b52 100644 --- a/sieval/tasks/liveoibench_0shot_gen.py +++ b/sieval/tasks/liveoibench_0shot_gen.py @@ -19,6 +19,7 @@ AI-Generated Code - Claude Opus 4.5 (Anthropic) """ +import asyncio import json import os import time @@ -31,7 +32,11 @@ from sieval.community.liveoibench.code_extractor import CodeExtractor from sieval.community.liveoibench.payloads import load_code_bundle, load_subtasks from sieval.community.liveoibench.prompts import build_prompt -from sieval.community.liveoibench.rankings import score_contest +from sieval.community.liveoibench.rankings import ( + build_problem_to_contest_map, + resolve_contest_id, + score_contest, +) from sieval.community.liveoibench.scoring import interprete_task_result, total_points from sieval.core.models import ModelOutput from sieval.core.tasks import ( @@ -60,6 +65,29 @@ _UPSTREAM = "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12" +# Case count assumed when this process cannot read the test tree but the +# evaluator can. Only sizes the HTTP deadline, so it is deliberately above the +# largest published problem (429 cases, JOI-2024-JOI_spring-2024-sp_Table_Tennis; +# median 42): too generous costs a slow failure, too tight aborts a live grade. +_ASSUMED_CASES = 450 + + +def _cutoff(value) -> float | None: + """A medal cutoff cell as a float, or ``None`` when the contest publishes none. + + Upstream spells this ``float(x) if not pd.isna(x) else None``. The distinction + matters to :func:`medal_from_cutoffs`: a NaN that survives as a float reads as + a published cutoff nothing can clear, so the contest reports the medal + ``"None"`` and joins the medal denominator instead of staying out of it. + """ + if value is None: + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return None if number != number else number + @sieval_task( name="liveoibench_0shot_gen", @@ -90,7 +118,18 @@ "to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 " "interactive ones are filtered out by the dataset (no interactor process " "yet), so numbers are over 380 problems and are NOT comparable to the " - "paper's 403-problem table. Codeforces Elo is not computed." + "paper's 403-problem table. Aggregation follows upstream's overall block: " + "relative_score is meaned within a contest and then across contests, " + "pass_rate is the fraction of problems fully solved, and tests_passed_pct " + "the fraction of test cases. Human ranking resolves a contest through the " + "contestant table's own 'problems' column, which is where USACO's division " + "split (-platinum / -combined) lives; all 22 USACO rows publish medal " + "cutoffs but no contestant list, so they yield a medal and no percentile, " + "and -combined yields neither -- upstream scores it from promotion " + "thresholds the published dataset does not carry. Three IATI problems " + "(senior-game, senior-ones, junior-twins) link against a grader whose " + "header the prompt never shows, so they cannot compile and score 0; " + "upstream behaves identically. Codeforces Elo is not computed." ), ), ) @@ -177,14 +216,18 @@ async def feedback(self, post, ctx): f"{raw['problem_id']}: no subtasks to score against" ) maximum = total_points(subtasks) - payload_tests = self._test_payload(raw) + # Both halves touch the test directory -- a `listdir`, and in inline mode + # ~140 MB of reads. Off the event loop, and once per sample rather than + # once per rollout. + payload_tests, n_cases = await asyncio.to_thread(self._test_payload, raw) + timeout = self._request_timeout(raw, n_cases) rollouts: list[RolloutJudgement] = [] best_index, best_key = None, None for rollout in post["rollouts"]: idx = rollout["index"] verdicts, case_names, msg, resources = await self._grade( - raw, rollout.get("prediction") or "", payload_tests, idx + raw, rollout.get("prediction") or "", payload_tests, idx, timeout ) results = [ @@ -252,31 +295,53 @@ async def feedback(self, post, ctx): }, ) - def _test_payload(self, raw) -> dict: - """The request's test half: a directory to read, or the cases inline.""" + def _test_payload(self, raw) -> tuple[dict, int]: + """The request's test half, and how many cases it names. + + A directory for the evaluator to read, or the cases inline. The count + only sizes the HTTP deadline — the evaluator decides the real case list. + Blocking; called from a worker thread. + """ tests_dir = raw["tests_dir"] + try: + stems = sorted( + name[: -len(".in")] + for name in os.listdir(tests_dir) + if name.endswith(".in") + ) + except OSError: + if self._inline_tests: + raise + # Directory mode only needs the evaluator to see the tree, not this + # process; fall back to a deadline wide enough for a large problem. + return {"test_dir": tests_dir}, _ASSUMED_CASES + if not self._inline_tests: - return {"test_dir": tests_dir} + return {"test_dir": tests_dir}, len(stems) inputs, outputs, names = [], [], [] - for filename in sorted(f for f in os.listdir(tests_dir) if f.endswith(".in")): - stem = filename[: -len(".in")] + for stem in stems: output_path = os.path.join(tests_dir, f"{stem}.out") if not os.path.exists(output_path): raise NonRetriableSampleError( f"{raw['problem_id']}: test {stem!r} has no .out in {tests_dir}" ) - with open( - os.path.join(tests_dir, filename), encoding="utf-8", errors="replace" - ) as f: + input_path = os.path.join(tests_dir, f"{stem}.in") + with open(input_path, encoding="utf-8", errors="replace") as f: inputs.append(f.read()) with open(output_path, encoding="utf-8", errors="replace") as f: outputs.append(f.read()) names.append(stem) - return {"test": {"inputs": inputs, "outputs": outputs, "names": names}} + payload = {"inputs": inputs, "outputs": outputs, "names": names} + return {"test": payload}, len(names) async def _grade( - self, raw, code: str, payload_tests: dict, idx: int + self, + raw, + code: str, + payload_tests: dict, + idx: int, + timeout: httpx.Timeout, ) -> tuple[list[bool], list[str], str, dict]: """One rollout through the evaluator; returns verdicts and their names.""" task_name = raw["task_name"] @@ -303,10 +368,7 @@ async def _grade( "memory_limit": int(raw["memory_limit"]), **payload_tests, }, - # Every case is bounded, so this only has to outlast the whole - # suite: the per-case wall the evaluator enforces is 120s, and a - # problem can carry a few hundred cases. - timeout=self._request_timeout(raw), + timeout=timeout, ) resp.raise_for_status() res = resp.json() @@ -333,6 +395,14 @@ async def _grade( f"(msg: {msg}); it needs the liveoibench C++ source " "(vendor/code-evaluator/VENDORED.md)." ) + if not case_names: + # An empty verdict vector scores every subtask at zero, so say what + # it is instead: the evaluator saw the directory and found nothing. + raise NonRetriableSampleError( + f"{raw['problem_id']}: the evaluator found no test cases in " + f"{raw['tests_dir']!r} (msg: {msg}). Re-run " + "`python scripts/materialize_liveoibench_tests.py --overwrite`." + ) resources = { key: value @@ -341,92 +411,160 @@ async def _grade( } return verdicts, case_names, msg, resources - def _request_timeout(self, raw) -> float: + def _request_timeout(self, raw, n_cases: int) -> httpx.Timeout: """HTTP deadline, outside every budget the evaluator can spend. Compilation is bounded at the evaluator's 60s default and each case at its own 120s wall, which only a process burning no CPU can reach. The suite runs at most 4 cases at a time there, so the bound is derived from the case count rather than assumed. + + ``pool=None`` because a bare float would also cap the wait for a free + connection, and these deadlines differ by an order of magnitude across + problems: a ten-case problem queued behind hour-long ones would fail on + the pool without ever being graded. """ - try: - n_cases = sum(1 for f in os.listdir(raw["tests_dir"]) if f.endswith(".in")) - except OSError: - n_cases = 100 per_case = min(120.0, float(raw["time_limit"]) * 1.2 + 5.0) - return 60.0 + per_case * n_cases + 30.0 + return httpx.Timeout(60.0 + per_case * n_cases + 30.0, pool=None) @override async def report(self, finals, fails): total = len(finals) + len(fails) - if not finals: - return { - "score": 0.0, - "fails": float(len(fails)), - "relative_score": 0.0, - SCORE_KEY_FIELD: "relative_score", - DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, - } - - # A failed sample scores 0 rather than dropping out of the mean, so the - # denominator is every sample requested. + contestants = self._load_contestants() + by_contest = self._group_by_contest(finals, fails, contestants) + + # Upstream's `overall` block (generate_rankings.py): `relative_score` is + # meaned inside a contest and then across contests, so a 12-problem round + # does not outweigh a 2-problem one. Problem counts here run 1..12, and + # the eleven largest rounds are all USACO, so the two weightings are not + # interchangeable. + contest_relative = [ + sum(r["relative_score"] for r in rows) / len(rows) + for rows in by_contest.values() + ] + contest_tests = [ + sum(r["tests_passed_pct"] for r in rows) / len(rows) + for rows in by_contest.values() + ] relative = ( - sum(f.feedback_result["metrics"]["relative_score"] for f in finals) / total - ) - pass_rate = ( - sum(f.feedback_result["metrics"]["tests_passed_pct"] for f in finals) - / total - ) - ace_rate = ( - sum(1 for f in finals if f.feedback_result["metrics"]["ace"]) / total * 100 + sum(contest_relative) / len(contest_relative) if contest_relative else 0.0 ) + # Upstream's `pass_rate` is problem-level -- `total_solved / + # total_problem_count` -- and counts a problem only when every test in it + # passed. A failed sample counts as unsolved rather than dropping out of + # the denominator, which is what DENOMINATOR_REQUESTED means here. + solved = sum(1 for rows in by_contest.values() for r in rows if r["solved"]) metrics: dict[str, float | str] = { "score": relative, "fails": float(len(fails)), "relative_score": relative, - "pass_rate": pass_rate, - "ace_rate": ace_rate, - "n_problems": float(len(finals)), + "pass_rate": (solved / total * 100) if total else 0.0, + "tests_passed_pct": ( + sum(contest_tests) / len(contest_tests) if contest_tests else 0.0 + ), + "n_problems": float(total), + "n_scored": float(len(finals)), + "n_contests": float(len(by_contest)), SCORE_KEY_FIELD: "relative_score", DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, } - metrics |= self._human_metrics(finals) + metrics |= self._human_metrics(by_contest, contestants) metrics |= budget_metrics( - [f.feedback_result["n_rollouts"] for f in finals], n=self._n, k=self._n + [f.feedback_result["n_rollouts"] for f in finals], + n=self._n, + # `k = n`: the headline is best-of-n, not pass@k, so every rollout + # drawn is a candidate for the one that scores. + k=self._n, ) return metrics | health_metrics(finals) - def _human_metrics(self, finals) -> dict[str, float]: + def _group_by_contest(self, finals, fails, contestants) -> dict[str, list[dict]]: + """Every requested problem, keyed by the contest it is *ranked* in. + + Not the contest its id names: USACO splits one round into a ``-platinum`` + and a ``-combined`` ranking, and only the contestant table says which + problems went where. :func:`resolve_contest_id` walks upstream's ladder, + falling back to the id-derived contest when the table has nothing. + + Failed samples ride along at zero — the denominator is every sample + requested, so a sample that never produced a judgement has to depress its + contest's mean rather than vanish from it. + """ + problem_to_contest = build_problem_to_contest_map(contestants.values()) + available = set(contestants) + + grouped: dict[str, list[dict]] = defaultdict(list) + unplaced = 0 + + def place(ctx, result) -> None: + nonlocal unplaced + raw = ctx.raw_sample + if raw is None: + unplaced += 1 + return + contest_id = ( + resolve_contest_id( + raw["problem_id"], raw["contest_id"], problem_to_contest, available + ) + or raw["contest_id"] + ) + grouped[contest_id].append( + { + "task_name": raw["task_name"], + "score": float(result["score"]) if result else 0.0, + "relative_score": ( + result["metrics"]["relative_score"] if result else 0.0 + ), + "tests_passed_pct": ( + result["metrics"]["tests_passed_pct"] if result else 0.0 + ), + "solved": bool(result["metrics"]["ace"]) if result else False, + } + ) + + for ctx in finals: + place(ctx, ctx.feedback_result) + for ctx in fails: + # A failed context never contributes a judgement, even when it + # produced one before failing: the run did not accept it. + place(ctx, None) + + if unplaced: + logger.warning( + "{} sample(s) carried no dataset row and are absent from the " + "per-contest means; they still count in the denominator.", + unplaced, + ) + return dict(grouped) + + def _human_metrics(self, by_contest, contestants) -> dict[str, float]: """Percentile and medals, per contest, against that contest's field. Upstream re-totals the humans over exactly the tasks the model was scored on, so a contest whose interactive problems were filtered out still compares like with like. Contests are averaged unweighted, as upstream averages its per-contest percentiles. + + The two counts are always reported once a contestant table was read, so + ``n_contests_ranked`` of 0 reads as measured rather than as missing — + every USACO contest in the release publishes cutoffs but no contestant + list, so a USACO-only run legitimately ranks nothing and still medals. """ - contestants = self._load_contestants() if not contestants: return {} - by_contest: dict[str, dict[str, float]] = defaultdict(dict) - for final in finals: - extra = final.feedback_result["extra"] - by_contest[extra["contest_id"]][extra["task_name"]] = final.feedback_result[ - "score" - ] - percentiles: list[float] = [] medals: list[str] = [] - unmatched = 0 - for contest_id, model_scores in by_contest.items(): + unmatched: list[str] = [] + for contest_id, rows in by_contest.items(): contest = contestants.get(contest_id) if contest is None: - unmatched += 1 + unmatched.append(contest_id) continue result = score_contest( contest["rankings"], - model_scores, + {row["task_name"]: row["score"] for row in rows}, contest_id=contest_id, gold_cutoff=contest["gold_cutoff"], silver_cutoff=contest["silver_cutoff"], @@ -439,15 +577,18 @@ def _human_metrics(self, finals) -> dict[str, float]: if unmatched: logger.warning( - "{} contest(s) had no contestant record; excluded from the " - "human metrics.", - unmatched, + "{} contest(s) had no contestant record and are excluded from the " + "human metrics: {}", + len(unmatched), + ", ".join(sorted(unmatched)), ) - metrics: dict[str, float] = {"n_contests": float(len(by_contest))} + metrics: dict[str, float] = { + "n_contests_ranked": float(len(percentiles)), + "n_contests_medalled": float(len(medals)), + } if percentiles: metrics["human_percentile"] = sum(percentiles) / len(percentiles) - metrics["n_contests_ranked"] = float(len(percentiles)) if medals: metrics["medal_rate"] = ( sum(1 for m in medals if m in ("Gold", "Silver", "Bronze")) @@ -479,10 +620,14 @@ def _load_contestants(self) -> dict[str, dict]: except json.JSONDecodeError: continue contestants[row["contest_id"]] = { + # `contest_id` and `problems` are what builds the problem-to-contest + # map; upstream reads the division split out of the same column. + "contest_id": row["contest_id"], + "problems": row.get("problems"), "rankings": rankings, - "gold_cutoff": row["gold_cutoff"], - "silver_cutoff": row["silver_cutoff"], - "bronze_cutoff": row["bronze_cutoff"], + "gold_cutoff": _cutoff(row["gold_cutoff"]), + "silver_cutoff": _cutoff(row["silver_cutoff"]), + "bronze_cutoff": _cutoff(row["bronze_cutoff"]), } return contestants diff --git a/tests/unit/community/liveoibench/test_rankings.py b/tests/unit/community/liveoibench/test_rankings.py index 8edb3f16..5487e36c 100644 --- a/tests/unit/community/liveoibench/test_rankings.py +++ b/tests/unit/community/liveoibench/test_rankings.py @@ -8,11 +8,16 @@ AI-Generated Code - Claude Opus 4.5 (Anthropic) """ +import json + from sieval.community.liveoibench.rankings import ( + build_problem_to_contest_map, calculate_percentile, identify_task_columns, medal_from_cutoffs, + normalize_contest_identifier, normalize_name, + resolve_contest_id, score_contest, ) @@ -144,3 +149,161 @@ def test_medals_use_the_contests_published_cutoffs(): ) assert result["medal"] == "Gold" assert result["n_contestants"] == 4 + + +# --------------------------------------------------------------------------- # +# Contest identity — USACO splits one round into two rankings +# --------------------------------------------------------------------------- # +def test_the_contest_map_reads_the_division_split_out_of_the_problems_column(): + """The only place the split is stated is the contestant row's own list.""" + rows = [ + { + "contest_id": "USACO-2025-January_Contest-platinum", + "problems": json.dumps(["USACO-2025-January_Contest-platinum_Cow"]), + }, + { + "contest_id": "USACO-2025-January_Contest-combined", + "problems": json.dumps( + [ + "USACO-2025-January_Contest-bronze_Moo", + "USACO-2025-January_Contest-gold_Hay", + ] + ), + }, + {"contest_id": "IOI-2025-contest", "problems": ["IOI-2025-contest-beechtree"]}, + ] + mapping = build_problem_to_contest_map(rows) + assert ( + mapping["USACO-2025-January_Contest-platinum_Cow"] + == "USACO-2025-January_Contest-platinum" + ) + assert ( + mapping["USACO-2025-January_Contest-bronze_Moo"] + == "USACO-2025-January_Contest-combined" + ) + assert mapping["IOI-2025-contest-beechtree"] == "IOI-2025-contest" + + +def test_a_division_listed_by_the_wrong_row_is_not_mapped(): + """Upstream's filter: a `-combined` row never claims a platinum problem.""" + rows = [ + { + "contest_id": "USACO-2025-January_Contest-combined", + "problems": json.dumps(["USACO-2025-January_Contest-platinum_Cow"]), + } + ] + assert build_problem_to_contest_map(rows) == {} + + +def test_the_contestant_table_wins_over_the_id_derived_contest(): + """Upstream reads the mapping out of the table first and only derives one + when the table lists nothing. + + On the published data the two routes happen to agree everywhere, so nothing + else here would notice the table being ignored — but the table is the only + one of the two that can follow a release that regroups a round. + """ + mapping = build_problem_to_contest_map( + [{"contest_id": "IOI-2025-day2", "problems": ["IOI-2025-contest-beechtree"]}] + ) + assert ( + resolve_contest_id( + "IOI-2025-contest-beechtree", + "IOI-2025-contest", + mapping, + {"IOI-2025-day2", "IOI-2025-contest"}, + ) + == "IOI-2025-day2" + ) + + +def test_an_unlisted_usaco_problem_falls_back_to_its_division_suffix(): + resolved = resolve_contest_id( + "USACO-2025-January_Contest-platinum_Cow", + "USACO-2025-January_Contest", + {}, + {"USACO-2025-January_Contest-platinum"}, + ) + assert resolved == "USACO-2025-January_Contest-platinum" + + +def test_a_non_usaco_problem_resolves_to_the_contest_its_id_names(): + assert ( + resolve_contest_id("IOI-2025-contest-beechtree", "IOI-2025-contest", {}, set()) + == "IOI-2025-contest" + ) + + +def test_the_cco_rename_falls_back_to_the_identifier_the_table_actually_uses(): + """Upstream renames the CCO rounds and the published table does not, so the + ladder has to come back to the raw identifier or CCO stops matching.""" + raw = "CCO-2024-Canadian_Computing_Competition_Senior" + assert normalize_contest_identifier(raw) == "CCO-2024-Senior" + assert resolve_contest_id(f"{raw}-x", raw, {}, {raw}) == raw + + +# --------------------------------------------------------------------------- # +# Recalculated_Total — upstream prefers it wherever it exists +# --------------------------------------------------------------------------- # +def test_a_recalculated_total_column_replaces_the_per_task_sum(): + """The two are not the same number: the column is the contest's own + re-derivation, and on the published data it differs from the per-task sum + for 20-83% of contestants on each of the 10 contests that carry it.""" + rankings = [ + {"alpha": 0, "beta": 0, "Recalculated_Total": 300}, + {"alpha": 100, "beta": 100, "Recalculated_Total": 10}, + ] + result = score_contest(rankings, {"alpha": 100, "beta": 0}) + assert result["matched_columns"] == ["Recalculated_Total"] + # Against the column the model's 100 beats only the 10 -> 50th. Summing the + # task columns would have compared it against 0 and 200 instead; the column + # name above is what discriminates, the percentile pins the values. + assert result["human_percentile"] == 50.0 + + +def test_the_recalculated_branch_drops_unparseable_rows_rather_than_zeroing_them(): + """Upstream's `.dropna()`. Zeroing instead would invent contestants the + model outscores, which inflates the percentile.""" + rankings = [ + {"Recalculated_Total": 300}, + {"Recalculated_Total": None}, + {"Recalculated_Total": "n/a"}, + ] + result = score_contest(rankings, {"alpha": 10}) + # One usable contestant, who beat the model. Zeroing the other two would + # have reported 66.7 instead. + assert result["human_percentile"] == 0.0 + + +def test_rows_without_the_column_still_take_the_per_task_branch(): + result = score_contest(RANKINGS, {"alpha": 100, "beta": 100}) + assert result["matched_columns"] == ["alpha", "beta"] + + +# --------------------------------------------------------------------------- # +# Contests that publish cutoffs but no contestants +# --------------------------------------------------------------------------- # +def test_an_empty_field_still_awards_the_medal_its_cutoffs_imply(): + """Upstream's `df.empty` branch. Every USACO row in the release is this + shape, so reporting no medal here would lose all 22 of them.""" + result = score_contest( + [], {"alpha": 100}, gold_cutoff=90.0, silver_cutoff=60.0, bronze_cutoff=30.0 + ) + assert result["human_percentile"] is None + assert result["medal"] == "Gold" + assert result["n_contestants"] == 0 + + +def test_a_usaco_combined_contest_reports_nothing(): + """Its metric is a promotion threshold that the published dataset does not + carry; upstream returns no medal and no percentile for it.""" + result = score_contest( + [], + {"bronze_Moo": 100}, + contest_id="USACO-2025-January_Contest-combined", + gold_cutoff=800.0, + silver_cutoff=750.0, + bronze_cutoff=700.0, + ) + assert result["human_percentile"] is None + assert result["medal"] is None diff --git a/tests/unit/datasets/test_liveoibench.py b/tests/unit/datasets/test_liveoibench.py index c1bda5df..64bd0f1e 100644 --- a/tests/unit/datasets/test_liveoibench.py +++ b/tests/unit/datasets/test_liveoibench.py @@ -134,6 +134,7 @@ def test_load_joins_subtasks_and_derives_the_columns_the_task_reads(tmp_path): path = _stage(tmp_path, [_problem(BATCH_ID)]) dataset = LiveOIBenchDataset(name_or_path=path) + assert dataset.test_set is not None (row,) = list(dataset.test_set) assert json.loads(row["subtasks"]) == SUBTASKS assert row["contest_id"] == "IOI-2025-contest" @@ -155,18 +156,21 @@ def test_interactive_problems_are_filtered_out_by_default(tmp_path): tmp_path, [_problem(BATCH_ID), _problem(INTERACTIVE_ID, task_type="interactive")], ) - assert [ - r["problem_id"] for r in LiveOIBenchDataset(name_or_path=path).test_set - ] == [BATCH_ID] + default = LiveOIBenchDataset(name_or_path=path).test_set + assert default is not None + assert [r["problem_id"] for r in default] == [BATCH_ID] both = LiveOIBenchDataset(name_or_path=path, task_type=None).test_set + assert both is not None assert len(both) == 2 def test_year_and_competition_filters_apply(tmp_path): path = _stage(tmp_path, [_problem(BATCH_ID), _problem(DASHED_ID)]) by_year = LiveOIBenchDataset(name_or_path=path, year="2024").test_set + assert by_year is not None assert [r["problem_id"] for r in by_year] == [DASHED_ID] by_competition = LiveOIBenchDataset(name_or_path=path, competition="IOI").test_set + assert by_competition is not None assert [r["problem_id"] for r in by_competition] == [BATCH_ID] @@ -185,6 +189,7 @@ def test_a_missing_test_corpus_names_the_command_that_builds_it(tmp_path): def test_prompts_can_be_loaded_without_the_corpus(tmp_path): path = _stage(tmp_path, [_problem(BATCH_ID)], with_tests=False) dataset = LiveOIBenchDataset(name_or_path=path, require_tests=False) + assert dataset.test_set is not None assert len(dataset.test_set) == 1 @@ -194,6 +199,7 @@ def test_an_external_tests_root_is_honoured(tmp_path): case_dir = problem_tests_dir(str(external), BATCH_ID) os.makedirs(case_dir) dataset = LiveOIBenchDataset(name_or_path=path, tests_root=str(external)) + assert dataset.test_set is not None assert list(dataset.test_set)[0]["tests_dir"] == case_dir diff --git a/tests/unit/scripts/test_materialize_liveoibench_tests.py b/tests/unit/scripts/test_materialize_liveoibench_tests.py new file mode 100644 index 00000000..bc8cc6b7 --- /dev/null +++ b/tests/unit/scripts/test_materialize_liveoibench_tests.py @@ -0,0 +1,93 @@ +"""Unit tests for unpacking the LiveOIBench test corpus. + +The load-bearing behaviour is atomicity. The unpack writes >30 GB and takes long +enough to be interrupted — Ctrl-C, a full disk, an OOM kill — and both the +script's own skip check and the dataset loader's ``require_tests`` guard read a +directory's *existence* as "this problem is materialized". A problem written in +place would therefore come back from a crash looking complete, and the missing +cases would surface much later as a subtask naming a test that is not there. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] + + +def _load_module(): + """Import the script by path; `scripts/` is not a package.""" + path = ROOT / "scripts" / "materialize_liveoibench_tests.py" + spec = importlib.util.spec_from_file_location("_materialize_liveoibench", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +materialize = _load_module() + +TESTS = { + "a1": {"input": "1\n", "output": "2\n"}, + "a2": {"input": "3\n", "output": "4\n"}, +} + + +def test_a_problems_cases_land_in_the_upstream_layout(tmp_path): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + materialize.write_problem_tests(out_dir, TESTS) + assert sorted(p.name for p in out_dir.iterdir()) == [ + "a1.in", + "a1.out", + "a2.in", + "a2.out", + ] + assert (out_dir / "a2.out").read_text() == "4\n" + + +def test_an_interrupted_problem_leaves_no_directory_to_mistake_for_a_whole_one( + tmp_path, monkeypatch +): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + + real_write = Path.write_text + state = {"n": 0} + + def exploding_write(self, data, *args, **kwargs): + state["n"] += 1 + if state["n"] > 2: + raise KeyboardInterrupt("simulated Ctrl-C mid-problem") + return real_write(self, data, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", exploding_write) + with pytest.raises(KeyboardInterrupt): + materialize.write_problem_tests(out_dir, TESTS) + + # The guard that matters: nothing named `tests` exists, so neither the + # script's skip check nor `require_tests` reads this problem as done. + assert not out_dir.exists() + assert out_dir.with_name("tests.partial").is_dir() + + +def test_a_crashed_partial_is_discarded_rather_than_merged(tmp_path): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + stale = out_dir.with_name("tests.partial") + stale.mkdir(parents=True) + (stale / "ghost.in").write_text("from an earlier crash\n") + + materialize.write_problem_tests(out_dir, TESTS) + assert not (out_dir / "ghost.in").exists() + assert not stale.exists() + + +def test_overwriting_replaces_the_whole_directory(tmp_path): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + materialize.write_problem_tests(out_dir, {"old": {"input": "x", "output": "y"}}) + materialize.write_problem_tests(out_dir, TESTS) + assert not (out_dir / "old.in").exists() + assert (out_dir / "a1.in").read_text() == "1\n" diff --git a/tests/unit/tasks/test_liveoibench_0shot_gen.py b/tests/unit/tasks/test_liveoibench_0shot_gen.py index d4d6eef8..600b9ddc 100644 --- a/tests/unit/tasks/test_liveoibench_0shot_gen.py +++ b/tests/unit/tasks/test_liveoibench_0shot_gen.py @@ -246,7 +246,12 @@ async def test_the_http_deadline_outlasts_every_budget_the_evaluator_can_spend( _raw(tests_dir=str(tmp_path)), ["int main(){}"], [[True, True, True]] ) # 60s compile bound + 3 cases * (1.5 * 1.2 + 5) + 30s slack. - assert evaluator.deadlines[0] == pytest.approx(60.0 + 3 * 6.8 + 30.0) + deadline = evaluator.deadlines[0] + assert deadline.read == pytest.approx(60.0 + 3 * 6.8 + 30.0) + # `pool` is deliberately unset: these deadlines differ by an order of + # magnitude across problems, and a bare float would make a small problem + # queued behind large ones fail while waiting for a connection. + assert deadline.pool is None # --------------------------------------------------------------------------- # @@ -329,6 +334,39 @@ async def post(self, url, *, json, timeout): await task.shutdown() +@pytest.mark.anyio +async def test_an_empty_test_directory_names_itself_instead_of_scoring_zero(): + """A half-written materialized problem would otherwise read as a submission + that failed every subtask, since an empty verdict vector scores nothing.""" + + class _NoCases(_Evaluator): + async def post(self, url, *, json, timeout): + self.bodies.append(json) + return _Response( + { + "status": False, + "msg": "no test cases to run", + "data": { + "n_cases": 0, + "n_passed": 0, + "case_verdicts": [], + "case_names": [], + "peak_memory_mb": 0.0, + }, + } + ) + + task = _task(_raw(), _NoCases([])) + try: + with pytest.raises(NonRetriableSampleError, match="no test cases"): + await task.feedback( + build_prediction_record(["int main(){}"]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + + @pytest.mark.anyio async def test_a_problem_with_no_rubric_fails_the_sample_without_retrying(): task = _task(_raw(subtasks="{}")) @@ -346,66 +384,132 @@ async def test_a_problem_with_no_rubric_fails_the_sample_without_retrying(): # Report # --------------------------------------------------------------------------- # class _Final: - """A finalized sample as `report` reads it: the judgement plus the - prediction record `health_metrics` counts extraction misses from.""" + """A finalized sample as `report` reads it: the dataset row it came from + (which names its contest), the judgement, and the prediction record + `health_metrics` counts extraction misses from.""" - def __init__(self, judgement: dict, predictions: list[str | None] | None = None): + def __init__( + self, + judgement: dict, + predictions: list[str | None] | None = None, + raw: dict | None = None, + ): + self.raw_sample = raw if raw is not None else _raw() self.feedback_result = judgement self.postprocess_result = build_prediction_record( predictions if predictions is not None else ["int main(){}"] ) +class _Fail: + """A sample that never produced a judgement. It still carries its dataset + row, so it can depress the mean of the contest it belongs to.""" + + def __init__(self, raw: dict | None = None): + self.raw_sample = raw if raw is not None else _raw() + self.feedback_result = None + self.postprocess_result = None + + +def _contestant_table(path, rows: list[dict]): + import pyarrow as pa + import pyarrow.parquet as pq + + pq.write_table(pa.Table.from_pylist(rows), path) + return path + + +def _contest_row( + contest_id: str, problems: list[str], ranking: list[dict], **overrides +): + return { + "contest_id": contest_id, + "problems": json.dumps(problems), + "gold_cutoff": 90.0, + "silver_cutoff": 60.0, + "bronze_cutoff": 30.0, + "contestants_ranking": json.dumps(ranking), + } | overrides + + @pytest.mark.anyio async def test_the_report_averages_over_every_requested_sample(): task = _task(_raw()) judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) try: - # One perfect problem plus one pipeline failure -> 50, not 100. - report = await task.report([_Final(judgement)], [object()]) + # One perfect problem plus one pipeline failure -> 50, not 100. Both are + # in the same contest, so the contest mean is the problem mean here. + report = await task.report([_Final(judgement)], [_Fail()]) finally: await task.shutdown() assert report["relative_score"] == 50.0 assert report["score"] == report["relative_score"] assert report["score_key"] == "relative_score" assert report["denominator_policy"] == "requested" - assert report["ace_rate"] == 50.0 + # Upstream's names: `pass_rate` counts problems fully solved, and a failed + # sample is not one of them. + assert report["pass_rate"] == 50.0 + assert report["tests_passed_pct"] == 50.0 assert report["fails"] == 1.0 + assert report["n_problems"] == 2.0 + assert report["n_scored"] == 1.0 + + +@pytest.mark.anyio +async def test_relative_score_weights_contests_equally_not_problems(): + """Upstream means inside a contest, then across contests. + + A 1-problem contest and a 3-problem contest therefore weigh the same. Under + problem-weighting the perfect singleton would pull the headline to 25.0. + """ + solved, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + big = [ + _Fail(_raw(problem_id="IOI-2025-contest-a", task_name="a")), + _Fail(_raw(problem_id="IOI-2025-contest-b", task_name="b")), + _Fail(_raw(problem_id="IOI-2025-contest-c", task_name="c")), + ] + small = _Final( + solved, + raw=_raw(problem_id="APIO-2025-contest-x", contest_id="APIO-2025-contest"), + ) + task = _task(_raw()) + try: + report = await task.report([small], big) + finally: + await task.shutdown() + assert report["relative_score"] == 50.0 + assert report["n_contests"] == 2.0 + # `pass_rate` stays problem-level, which is where the two differ. + assert report["pass_rate"] == 25.0 @pytest.mark.anyio async def test_an_empty_run_still_declares_its_headline(): task = _task(_raw()) try: - report = await task.report([], [object()]) + report = await task.report([], [_Fail()]) finally: await task.shutdown() assert report["score"] == 0.0 assert report["score_key"] == "relative_score" assert report["denominator_policy"] == "requested" + # The all-fail report carries the same columns as any other, so a reader + # does not KeyError on exactly the run that went worst. + for key in ("relative_score", "pass_rate", "tests_passed_pct", "n_problems"): + assert key in report @pytest.mark.anyio async def test_human_metrics_come_from_the_staged_contestant_table(tmp_path): - import pyarrow as pa - import pyarrow.parquet as pq - - contestants = tmp_path / "contest_results.parquet" - pq.write_table( - pa.Table.from_pylist( - [ - { - "contest_id": "IOI-2025-contest", - "gold_cutoff": 90.0, - "silver_cutoff": 60.0, - "bronze_cutoff": 30.0, - "contestants_ranking": json.dumps( - [{"Rank": 1, "beechtree": 100}, {"Rank": 2, "beechtree": 40}] - ), - } - ] - ), - contestants, + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "IOI-2025-contest", + [PROBLEM_ID], + [{"Rank": 1, "beechtree": 100}, {"Rank": 2, "beechtree": 40}], + ) + ], ) judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) @@ -420,6 +524,87 @@ async def test_human_metrics_come_from_the_staged_contestant_table(tmp_path): assert report["human_percentile"] == 50.0 assert report["gold_rate"] == 100.0 assert report["n_contests"] == 1.0 + assert report["n_contests_ranked"] == 1.0 + + +@pytest.mark.anyio +async def test_usaco_problems_rank_against_their_division_row(tmp_path): + """The contestant table keys USACO by division, not by the round in the id. + + Without the routing every USACO problem misses the join and drops out of the + human metrics entirely — 132 of the 380 scored problems on the real data. + """ + platinum_id = "USACO-2025-January_Contest-platinum_Cowpatibility" + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "USACO-2025-January_Contest-platinum", + [platinum_id], + [], # every published USACO row ships cutoffs and no ranking + gold_cutoff=250.0, + silver_cutoff=150.0, + bronze_cutoff=50.0, + ) + ], + ) + raw = _raw( + problem_id=platinum_id, + task_name="platinum_Cowpatibility", + contest_id="USACO-2025-January_Contest", + ) + judgement, _ = await _grade(raw, ["int main(){}"], [[True, True, True]]) + task = _task(raw) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement, raw=raw)], []) + finally: + await task.shutdown() + + # Matched: the contest publishes no contestants, so there is no percentile, + # but the medal cutoffs still apply (upstream's `df.empty` branch). + assert "human_percentile" not in report + assert report["n_contests_ranked"] == 0.0 + assert report["n_contests_medalled"] == 1.0 + assert report["medal_rate"] == 100.0 + + +@pytest.mark.anyio +async def test_a_usaco_combined_contest_reports_neither_percentile_nor_medal(tmp_path): + """Upstream scores `-combined` from promotion thresholds that ship in its + own repository, not in the dataset, so it computes nothing here either.""" + bronze_id = "USACO-2025-January_Contest-bronze_Cowmpetition" + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "USACO-2025-January_Contest-combined", + [bronze_id], + [], + gold_cutoff=800.0, + silver_cutoff=750.0, + bronze_cutoff=700.0, + ) + ], + ) + raw = _raw( + problem_id=bronze_id, + task_name="bronze_Cowmpetition", + contest_id="USACO-2025-January_Contest", + ) + judgement, _ = await _grade(raw, ["int main(){}"], [[True, True, True]]) + task = _task(raw) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement, raw=raw)], []) + finally: + await task.shutdown() + + assert "human_percentile" not in report + assert "medal_rate" not in report + assert report["n_contests_medalled"] == 0.0 + # The problem is still scored; only its human comparison is unavailable. + assert report["relative_score"] == 100.0 @pytest.mark.anyio @@ -433,6 +618,9 @@ async def test_a_missing_contestant_table_leaves_the_scores_reportable(tmp_path) await task.shutdown() assert report["relative_score"] == 100.0 assert "human_percentile" not in report + # No table read at all, so not even the counts are reported: a 0 here would + # read as "measured, nothing ranked". + assert "n_contests_ranked" not in report @pytest.mark.anyio diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index 870a5e04..f5a0fdef 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -491,12 +491,17 @@ Two kinds, and the difference is a decision rather than a status: single-number outputs compare at `rel_tol=abs_tol=1e-6`; else exact; else line count plus stripped per-line match). - Three differences from upstream LiveOIBench, all documented in the module: + Four differences from upstream LiveOIBench, all documented in the module: compilation is bounded (`timeout`, default 60 s) where upstream's `subprocess.run` has no timeout; every test always runs, since subtask scoring - needs the whole verdict vector; and no checker path exists, because the - published dataset ships no `checkers/` directory, so upstream's own judge - compares outputs directly on this data. + needs the whole verdict vector; no checker path exists, because the published + dataset ships no `checkers/` directory, so upstream's own judge compares + outputs directly on this data; and a suite with **no** test case is refused + by name (`status=False`, `"no test cases to run"`) rather than graded. That + last one is a deployment fault — a half-written materialized directory — and + the two readings it otherwise gets are both wrong: an empty verdict vector + scores every subtask at zero, and the failure message has no first failure to + name, which raised `StopIteration` out of a coroutine. `Sample` gains `files` (extra sources compiled alongside — `grader.cpp`, `{task}.h`) and `entry_filename`; the test model gains `names`, used only in diff --git a/vendor/code-evaluator/app/exec_cpp.py b/vendor/code-evaluator/app/exec_cpp.py index 7239c0c7..3d21efff 100644 --- a/vendor/code-evaluator/app/exec_cpp.py +++ b/vendor/code-evaluator/app/exec_cpp.py @@ -319,7 +319,9 @@ def build_cases( input_data=input_data, expected=expected, ) - for i, (input_data, expected) in enumerate(zip(inputs, expect_outputs)) + for i, (input_data, expected) in enumerate( + zip(inputs, expect_outputs, strict=True) + ) ] @@ -351,6 +353,13 @@ async def execute_tests( files = dict(files or {}) stats = ResourceStats() + if not cases: + # A problem with no test case is a deployment fault -- an empty or + # half-written materialized directory -- not a verdict on the submission. + # Said here rather than left to the failure message below, which would + # otherwise look for a first failure among no results and raise. + return False, "no test cases to run", stats, 0, [] + with tempfile.TemporaryDirectory(prefix="liveoibench-") as workdir: for filename, contents in files.items(): # Flat by construction: upstream's bundles are `grader.cpp` / `task.h`. From 27f5cbc6033a3d03c2ac2b5e1a4460ea726c4a5a Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 15:34:22 +0800 Subject: [PATCH 3/5] chore(vendor): drop Dockerfile.cpp; the MultiPL-E image already serves liveoibench `Dockerfile.multipl-e`, which landed on main alongside the table-driven language support, was byte-identical to `Dockerfile.cpp` apart from its extra toolchains: same base, same workdir, same requirements, same `COPY app/`, same CMD and port. The only question was whether its `apt-get install g++` covers what the liveoibench `-static` link needs without naming `libstdc++-*-dev` and `libc6-dev` the way `Dockerfile.cpp` did. It does, and this was measured inside the real base rather than reasoned about. On the current `python:3.10-slim` (Debian 13.6), `g++` alone brings in `libc6-dev 2.41` and `libstdc++-14-dev`; `libstdc++.a` and `libc.a` both resolve; and the judge's exact command line -- `g++ -std=gnu++17 -Wall -O2 -pipe -static -g` -- builds and runs a `bits/stdc++.h` program. Debian's `g++` metapackage depends on the versioned compiler, which depends on the matching `libstdc++-N-dev`, which depends on `libc6-dev`, so the two explicit packages were never adding anything. Worse, they had started to drift: `Dockerfile.cpp` pinned `libstdc++-12-dev` while the base has since moved to gcc 14, so it installed a dev tree the compiler no longer uses. A second image to build, deploy and keep current, for nothing. `Dockerfile.multipl-e`'s toolchain comment now records that g++ has a second consumer -- `source="liveoibench"` reaches a different C++ path (`exec_cpp.py`: per test case, `-static`, under the problem's own RLIMIT_CPU/RLIMIT_AS) than the `cpp` row in `exec_lang.LANGUAGES` -- so a later edit does not drop the compiler on the grounds that MultiPL-E is the only caller. `VENDORED.md` and `examples/liveoibench.yaml` name the surviving image. Preflight 25/25, no warnings. Co-Authored-By: Claude Opus 5 (1M context) --- examples/liveoibench.yaml | 5 +-- vendor/code-evaluator/VENDORED.md | 36 ++++++++++++------- vendor/code-evaluator/docker/Dockerfile.cpp | 20 ----------- .../docker/Dockerfile.multipl-e | 7 ++++ 4 files changed, 33 insertions(+), 35 deletions(-) delete mode 100644 vendor/code-evaluator/docker/Dockerfile.cpp diff --git a/examples/liveoibench.yaml b/examples/liveoibench.yaml index 60728c18..dcffb75c 100644 --- a/examples/liveoibench.yaml +++ b/examples/liveoibench.yaml @@ -19,8 +19,9 @@ # THIS NEEDS AN EVALUATOR WITH THE C++ SOURCE. Grading compiles the submission # with g++ and runs it under the problem's own time and memory limits, which the # base evaluator image cannot do — build it from -# `vendor/code-evaluator/docker/Dockerfile.cpp` and point SIEVAL_CODE_EVAL_API at -# it. The evaluator also reads the test directory off disk, so it needs the data +# `vendor/code-evaluator/docker/Dockerfile.multipl-e` (the image that carries a +# C++ toolchain; it serves this source too) and point SIEVAL_CODE_EVAL_API at it. +# The evaluator also reads the test directory off disk, so it needs the data # volume mounted; if it cannot see it, set `inline_tests: true` below and accept # ~140 MB per rollout on the wire. # diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index f5a0fdef..c96cbd19 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -481,8 +481,8 @@ Two kinds, and the difference is a decision rather than a status: -H "Accept: application/vnd.oci.image.manifest.v1+json" \ "https://ghcr.io/v2/nuprl/agnostics/manifests/lua" | grep -i docker-content-digest ``` -- `app/exec_cpp.py`, `app/server.py`, `docker/Dockerfile.cpp` — **C++ execution**, - for `source="liveoibench"`. The first non-Python language the evaluator runs as +- `app/exec_cpp.py`, `app/server.py` — **C++ execution**, for + `source="liveoibench"`. The first non-Python language the evaluator runs as a *compiled* artifact: `g++ -std=gnu++17 -Wall -O2 -pipe -static -g`, then one child per test case under `RLIMIT_CPU` / `RLIMIT_AS`. Both limits carry upstream LiveOIBench's explicit 20% buffer, and a 10 ms poller kills a child @@ -514,17 +514,27 @@ Two kinds, and the difference is a decision rather than a status: it; renaming it would widen the diff against upstream without changing the wire format. - `Dockerfile.cpp` exists because the base image has no toolchain: g++ without - `libstdc++-*-dev` / `libc6-dev` links nothing statically, and every submission - would fail identically. Verified against g++ 14.2 on Debian: correct, partial, - TLE, MLE, compile-error, float-tolerance and grader-linked submissions all - produce the expected verdict vectors. Not yet upstream — land in - `scitix/code-evaluator` and re-vendor. - - `Dockerfile.multipl-e` also serves this source — it installs `g++`, which - depends on `libstdc++-*-dev` and so pulls `libc6-dev` with it, the same two - packages named explicitly here. An operator running both benchmarks needs one - image, not two; this one stays as the minimal single-purpose build. + **Deploy `docker/Dockerfile.multipl-e`** — there is no separate image for this + source. The base image has no toolchain, and a `-static` link additionally + needs `libstdc++-*-dev` and `libc6-dev`, but `apt-get install g++` already + pulls both: Debian's `g++` metapackage depends on the versioned compiler, which + depends on the matching `libstdc++-N-dev`, which depends on `libc6-dev`. + Measured inside the real base rather than reasoned about — on the current + `python:3.10-slim` (Debian 13.6), `g++` alone brings `libc6-dev 2.41` and + `libstdc++-14-dev`, `libstdc++.a` and `libc.a` both resolve, and the judge's + exact link (`g++ -std=gnu++17 -Wall -O2 -pipe -static -g`) builds and runs a + `bits/stdc++.h` program. + + An earlier `Dockerfile.cpp` named those two packages explicitly and was + otherwise byte-identical to `Dockerfile.multipl-e` minus its other toolchains. + It was removed: it pinned `libstdc++-12-dev` while the base has since moved to + gcc 14, so it installed a dev tree the compiler no longer uses — a second image + to maintain that was both redundant and drifting. + + Verified against g++ 14.2 on Debian: correct, partial, TLE, MLE, + compile-error, float-tolerance and grader-linked submissions all produce the + expected verdict vectors. Not yet upstream — land in `scitix/code-evaluator` + and re-vendor. **Not the same C++ path as `exec_lang`'s `cpp` row above**, and the two are not merge candidates. That one is direct-run: one program, one all-or-nothing diff --git a/vendor/code-evaluator/docker/Dockerfile.cpp b/vendor/code-evaluator/docker/Dockerfile.cpp deleted file mode 100644 index 5e52fc57..00000000 --- a/vendor/code-evaluator/docker/Dockerfile.cpp +++ /dev/null @@ -1,20 +0,0 @@ -# Image for the `liveoibench` source: the base image has no C++ toolchain, and -# `exec_cpp` shells out to `g++`. `libstdc++-*-dev` and `libc6-dev` are what the -# upstream judge's `-static` link needs; without them g++ is present but every -# submission fails to link, which reads as a benchmark of nothing. -FROM python:3.10-slim - -WORKDIR /opt/code-evaluator - -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - libc6-dev \ - libstdc++-12-dev \ - && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt - -COPY app/ ./app/ - -CMD ["fastapi", "run", "app/server.py", "--port", "11451"] diff --git a/vendor/code-evaluator/docker/Dockerfile.multipl-e b/vendor/code-evaluator/docker/Dockerfile.multipl-e index 60f5bf89..8a4e90fc 100644 --- a/vendor/code-evaluator/docker/Dockerfile.multipl-e +++ b/vendor/code-evaluator/docker/Dockerfile.multipl-e @@ -7,6 +7,13 @@ FROM python:3.10-slim # # g++ -- humaneval-cpp / mbpp-cpp (build step; `bits/stdc++.h`, which # every cpp prompt includes, is GCC's) +# AND source="liveoibench" (app/exec_cpp.py, a different C++ path: +# per-test-case, `-static`, under the +# problem's own RLIMIT_CPU/RLIMIT_AS. +# It needs `libstdc++-*-dev` + +# `libc6-dev` to link statically, which +# `g++` already depends on -- verified in +# this base, do not "simplify" g++ away) # bash -- humaneval-sh / mbpp-sh (in the base image; named here so a # base change that drops it is visible) # perl -- humaneval-pl / mbpp-pl From 97ae2102f88a3e4548474fe0e863bedbcf24e730 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 19:04:37 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix(tasks):=20review=20fixes=20for=20LiveOI?= =?UTF-8?q?Bench=20=E2=80=94=20status,=20ranking=20scope,=20guards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the rebased branch. Nothing in the port's arithmetic moved: the extractor is still byte-identical to upstream below its provenance header, and `score_contest` still agrees with upstream's pandas `compute_human_metrics` on every contest of the published table (360 contest x score-level pairs, zero mismatches). `sieval/community/liveoibench/` is untouched by this commit. * **`status` is `experimental`, not `stable`.** In-tree the word means one thing -- faithful port, published anchor unverified or unreachable -- and this is that: no model has been run, the paper's table is over 403 problems where this scores 380, and upstream publishes no model outputs (no `submission_results/`, no results CSV, no solutions repo) to anchor the grader against instead, so the quotebench route of anchoring at zero model cost is not available either. `agieval` is the precedent in the other direction: a full 7,272-row run and still `experimental`, because a clean run is not an anchor. The reason now ships in `reference_impl.notes` rather than living only in the PR description, which is not published. * **A failed sample no longer enters the human comparison.** It was placed in its contest at score 0 under its real task name, which matched that problem's human column -- so the model's total lost the points *and* the human totals gained a column for a problem that was never graded. Upstream re-totals the humans over exactly the tasks the model was *scored* on and never writes a problem result for a failure. Failed samples still count at zero in `relative_score` and `pass_rate`, where the denominator is every requested sample. A contest whose problems all failed now reports no percentile instead of a percentile of zero. * **An empty rollout list is refused by name.** `next(r for r in rollouts ...)` over no rollouts raised `StopIteration` out of a coroutine, which Python re-raises as a bare "coroutine raised StopIteration" naming neither the sample nor the cause -- the same failure shape the evaluator's own empty-suite guard was added to avoid, one layer up. * **A task-name collision inside one contest is announced.** Keying the model's scores by task name is what matches a human column; two problems sharing a name would collapse into one entry and quietly drop the other from both the model total and the matched columns. No published contest does this (0 of 72), so it says so rather than scoring a smaller contest. * **A NULL `subtasks` cell stays absent** instead of being coerced to `"{}"`, which passed the loader's own broken-join check and only surfaced per sample at grading time. The "no test parquet found" guard now tracks whether a file was read rather than whether any rubric was: those are two different failures and the second was reporting the first's message. * `LiveOIBenchDatasetSample` declares the three columns it passes through but never reads (`id`, `setup_script`, `evaluation_script`); `VENDORED.md` states the trust boundary `test_dir` assumes, and why the two CPU fields are 0.0 on this source rather than missing. Verified: 7,417 unit tests pass (5 new), each fix reverse-mutated at its own site and all 5 mutants caught. ruff, ty and preflight (25/25, no warnings) all clean. Lock re-resolved with `--update-reuse`; the only change is the content hash. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/datasets/liveoibench.py | 22 +++- sieval/meta/index.json | 4 +- sieval/tasks/liveoibench_0shot_gen.py | 66 +++++++++- tests/unit/datasets/test_liveoibench.py | 17 +++ .../unit/tasks/test_liveoibench_0shot_gen.py | 118 ++++++++++++++++++ vendor/code-evaluator/VENDORED.md | 12 ++ 6 files changed, 229 insertions(+), 10 deletions(-) diff --git a/sieval/datasets/liveoibench.py b/sieval/datasets/liveoibench.py index 433d85de..196f12da 100644 --- a/sieval/datasets/liveoibench.py +++ b/sieval/datasets/liveoibench.py @@ -87,10 +87,18 @@ def problem_tests_dir(tests_root: str, problem_id: str) -> str: class LiveOIBenchDatasetSample(TypedDict): + # Every column the problems parquet carries, because rows are passed through + # whole (`row | {...}` below) rather than projected -- including the three + # this task never reads. `setup_script` / `evaluation_script` are non-empty + # only on the 5 script-judged problems, all of which are `interactive` and so + # are already filtered out by the default `task_type`. + id: int problem_id: str competition: str contest: str task_name: str + setup_script: str + evaluation_script: str problem_statement: str time_limit: float memory_limit: float @@ -244,18 +252,28 @@ def _read_subtasks(tests_repo: str) -> dict[str, str]: between a 20 KB read and the whole corpus. """ subtasks: dict[str, str] = {} + # Tracked separately from `subtasks` being non-empty: those are two different + # failures. No file is "the corpus was never downloaded"; a file whose rubrics + # are all absent is a join failure the loader below reports per problem. + found_a_parquet = False for year in TEST_YEARS: path = os.path.join(tests_repo, year_parquet_name(year)) if not os.path.exists(path): continue + found_a_parquet = True table = pq.read_table(path, columns=["problem_id", "subtasks"]) for problem_id, payload in zip( table.column("problem_id").to_pylist(), table.column("subtasks").to_pylist(), strict=True, ): - subtasks[problem_id] = payload or "{}" - if not subtasks: + # NOT `payload or "{}"`: a NULL cell would then read as a present but + # empty rubric, pass the loader's `is None` check below, and only + # surface per sample at grading time. Keep the absence, so the one + # place that decides what a missing rubric means is the loader. + if payload is not None: + subtasks[problem_id] = payload + if not found_a_parquet: raise FileNotFoundError( f"No LiveOIBench test parquet found under {tests_repo!r}; " "run `sieval dataset download liveoibench` first." diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 7d470495..57cccaed 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -2410,9 +2410,9 @@ "reference_impl": { "source": "liveoibench", "url": "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/base_judge.py", - "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Aggregation follows upstream's overall block: relative_score is meaned within a contest and then across contests, pass_rate is the fraction of problems fully solved, and tests_passed_pct the fraction of test cases. Human ranking resolves a contest through the contestant table's own 'problems' column, which is where USACO's division split (-platinum / -combined) lives; all 22 USACO rows publish medal cutoffs but no contestant list, so they yield a medal and no percentile, and -combined yields neither -- upstream scores it from promotion thresholds the published dataset does not carry. Three IATI problems (senior-game, senior-ones, junior-twins) link against a grader whose header the prompt never shows, so they cannot compile and score 0; upstream behaves identically. Codeforces Elo is not computed." + "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Aggregation follows upstream's overall block: relative_score is meaned within a contest and then across contests, pass_rate is the fraction of problems fully solved, and tests_passed_pct the fraction of test cases. Human ranking resolves a contest through the contestant table's own 'problems' column, which is where USACO's division split (-platinum / -combined) lives; all 22 USACO rows publish medal cutoffs but no contestant list, so they yield a medal and no percentile, and -combined yields neither -- upstream scores it from promotion thresholds the published dataset does not carry. Three IATI problems (senior-game, senior-ones, junior-twins) link against a grader whose header the prompt never shows, so they cannot compile and score 0; upstream behaves identically. Codeforces Elo is not computed. No published number has been reproduced, which is what keeps this 'experimental': the paper's table is over 403 problems where this scores 380, so it is not a reachable anchor, and upstream publishes no model outputs to grade against instead (no submission_results/, no results CSV, no solutions repo) -- so unlike quotebench there is no zero-cost grader anchor either. Promote to 'stable' once an n=8 run is aligned and the alignment card is filled in." }, - "status": "stable", + "status": "experimental", "reference_kind": "procedure" }, { diff --git a/sieval/tasks/liveoibench_0shot_gen.py b/sieval/tasks/liveoibench_0shot_gen.py index 54f83b52..fc25adea 100644 --- a/sieval/tasks/liveoibench_0shot_gen.py +++ b/sieval/tasks/liveoibench_0shot_gen.py @@ -101,6 +101,10 @@ def _cutoff(value) -> float | None: tags=("english", "cpp", "code-exec"), model_type="chat", reference_kind="procedure", + # Faithful port, anchor not yet reached -- the in-tree meaning of + # "experimental" (cf. agieval, which is validated on a full 7,272-row run and + # stays experimental because its anchor is unservable). See `notes` below. + status="experimental", reference_impl=ReferenceImpl( source="liveoibench", url=f"{_UPSTREAM}/src/judges/base_judge.py", @@ -129,7 +133,14 @@ def _cutoff(value) -> float | None: "thresholds the published dataset does not carry. Three IATI problems " "(senior-game, senior-ones, junior-twins) link against a grader whose " "header the prompt never shows, so they cannot compile and score 0; " - "upstream behaves identically. Codeforces Elo is not computed." + "upstream behaves identically. Codeforces Elo is not computed. " + "No published number has been reproduced, which is what keeps this " + "'experimental': the paper's table is over 403 problems where this " + "scores 380, so it is not a reachable anchor, and upstream publishes " + "no model outputs to grade against instead (no submission_results/, " + "no results CSV, no solutions repo) -- so unlike quotebench there is " + "no zero-cost grader anchor either. Promote to 'stable' once an n=8 " + "run is aligned and the alignment card is filled in." ), ), ) @@ -222,6 +233,16 @@ async def feedback(self, post, ctx): payload_tests, n_cases = await asyncio.to_thread(self._test_payload, raw) timeout = self._request_timeout(raw, n_cases) + if not post["rollouts"]: + # No rollout to pick a best from. Saying so by name is the point: + # the `next(...)` below would otherwise raise StopIteration out of a + # coroutine, which Python re-raises as a bare "coroutine raised + # StopIteration" naming neither the sample nor the cause -- the same + # failure shape the evaluator's empty-suite guard exists to avoid. + raise NonRetriableSampleError( + f"{raw['problem_id']}: inference returned no rollouts to grade" + ) + rollouts: list[RolloutJudgement] = [] best_index, best_key = None, None for rollout in post["rollouts"]: @@ -434,10 +455,11 @@ async def report(self, finals, fails): by_contest = self._group_by_contest(finals, fails, contestants) # Upstream's `overall` block (generate_rankings.py): `relative_score` is - # meaned inside a contest and then across contests, so a 12-problem round - # does not outweigh a 2-problem one. Problem counts here run 1..12, and - # the eleven largest rounds are all USACO, so the two weightings are not - # interchangeable. + # meaned inside a contest and then across contests, so a 10-problem round + # does not outweigh a 1-problem one. Measured over the 380 scored + # problems: 72 contests sized 1..10, the largest being + # JOI-2023-JOI_spring at 10, then eleven USACO rounds at 9 each. So the + # two weightings are not interchangeable. contest_relative = [ sum(r["relative_score"] for r in rows) / len(rows) for rows in by_contest.values() @@ -520,6 +542,10 @@ def place(ctx, result) -> None: result["metrics"]["tests_passed_pct"] if result else 0.0 ), "solved": bool(result["metrics"]["ace"]) if result else False, + # Whether this problem produced a judgement at all. The + # score-based means above take it at zero; the human + # comparison must not, see `_human_metrics`. + "scored": result is not None, } ) @@ -546,6 +572,15 @@ def _human_metrics(self, by_contest, contestants) -> dict[str, float]: compares like with like. Contests are averaged unweighted, as upstream averages its per-contest percentiles. + "Scored on" is why a **failed** sample is dropped here while it still + counts at zero in `relative_score` and `pass_rate`. Feeding it in would + penalise twice over: the model's total loses those points *and* the human + totals gain the column for a problem the model was never graded on. A + contest whose problems all failed therefore reports no percentile rather + than a percentile of zero, which is the honest reading -- nothing was + measured. Upstream reaches the same place by never writing a problem + result for it. + The two counts are always reported once a contestant table was read, so ``n_contests_ranked`` of 0 reads as measured rather than as missing — every USACO contest in the release publishes cutoffs but no contestant @@ -562,9 +597,28 @@ def _human_metrics(self, by_contest, contestants) -> dict[str, float]: if contest is None: unmatched.append(contest_id) continue + scored_rows = [row for row in rows if row["scored"]] + if not scored_rows: + continue + model_scores = {row["task_name"]: row["score"] for row in scored_rows} + if len(model_scores) != len(scored_rows): + # Upstream keys this by problem id and only maps to a task column + # late; keying by task name here is what lets one human column be + # matched once. Two problems sharing a name inside one contest + # would collapse into a single entry, dropping the other from both + # the model total and the matched columns. No contest in the + # published data does this (checked: 0 of 72), so this says so + # rather than scoring a quietly smaller contest. + logger.warning( + "{}: {} scored problems share only {} distinct task name(s); " + "the human comparison for this contest is incomplete.", + contest_id, + len(scored_rows), + len(model_scores), + ) result = score_contest( contest["rankings"], - {row["task_name"]: row["score"] for row in rows}, + model_scores, contest_id=contest_id, gold_cutoff=contest["gold_cutoff"], silver_cutoff=contest["silver_cutoff"], diff --git a/tests/unit/datasets/test_liveoibench.py b/tests/unit/datasets/test_liveoibench.py index 64bd0f1e..50a08f23 100644 --- a/tests/unit/datasets/test_liveoibench.py +++ b/tests/unit/datasets/test_liveoibench.py @@ -215,6 +215,23 @@ def test_a_problem_absent_from_the_test_parquet_is_a_loud_join_failure(tmp_path) LiveOIBenchDataset(name_or_path=path) +def test_a_null_subtasks_cell_is_the_same_loud_join_failure(tmp_path): + """A NULL rubric must not read as a present-but-empty one. + + Coerced to "{}" it passes the loader's own check and only surfaces later, + once per sample, at grading time -- where it looks like a scoring problem + rather than two parquets out of step. + """ + path = _stage(tmp_path, [_problem(BATCH_ID)]) + tests_repo = tmp_path / "LiveOIBench_tests" + pq.write_table( + pa.Table.from_pylist([{"id": 0, "problem_id": BATCH_ID, "subtasks": None}]), + tests_repo / year_parquet_name("2025"), + ) + with pytest.raises(ValueError, match="out of step"): + LiveOIBenchDataset(name_or_path=path) + + def test_a_missing_year_parquet_names_the_download_command(tmp_path): path = _stage(tmp_path, [_problem(BATCH_ID)]) os.remove(tmp_path / "LiveOIBench_tests" / year_parquet_name("2025")) diff --git a/tests/unit/tasks/test_liveoibench_0shot_gen.py b/tests/unit/tasks/test_liveoibench_0shot_gen.py index 600b9ddc..22cd89cb 100644 --- a/tests/unit/tasks/test_liveoibench_0shot_gen.py +++ b/tests/unit/tasks/test_liveoibench_0shot_gen.py @@ -623,6 +623,124 @@ async def test_a_missing_contestant_table_leaves_the_scores_reportable(tmp_path) assert "n_contests_ranked" not in report +@pytest.mark.anyio +async def test_a_failed_sample_is_not_compared_against_the_human_field(tmp_path): + """A failed sample counts at zero in the score means, never in the ranking. + + Upstream re-totals the humans over exactly the tasks the model was *scored* + on, and never writes a problem result for one that produced no judgement. + Feeding it in penalises twice: the model total loses the points and the human + total gains that problem's column. Here that is the whole gap between the + 100th percentile and the 0th. + """ + solved = _raw() + failed = _raw(problem_id="IOI-2025-contest-sphinx", task_name="sphinx") + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "IOI-2025-contest", + [solved["problem_id"], failed["problem_id"]], + # Both humans are weak on `beechtree` and strong on `sphinx`. + [ + {"beechtree": 50, "sphinx": 100}, + {"beechtree": 30, "sphinx": 90}, + ], + ) + ], + ) + + judgement, _ = await _grade(solved, ["int main(){}"], [[True, True, True]]) + task = _task(solved) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement)], [_Fail(failed)]) + finally: + await task.shutdown() + + # Scored on `beechtree` alone: 100 beats both 50 and 30 -> 100th percentile. + # Counting the failed `sphinx` at 0 would compare 100 against totals of + # 150/120 and report 0.0 instead. + assert report["human_percentile"] == 100.0 + assert report["n_contests_ranked"] == 1.0 + # The failure is still in the score means and the denominator. + assert report["relative_score"] == 50.0 + assert report["n_problems"] == 2.0 + + +@pytest.mark.anyio +async def test_a_contest_whose_problems_all_failed_reports_no_percentile(tmp_path): + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [_contest_row("IOI-2025-contest", [PROBLEM_ID], [{"beechtree": 50}])], + ) + task = _task(_raw()) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([], [_Fail()]) + finally: + await task.shutdown() + # Nothing was measured, so there is no percentile -- not a percentile of 0, + # which would read as "the model was beaten by the whole field". + assert "human_percentile" not in report + assert report["n_contests_ranked"] == 0.0 + + +@pytest.mark.anyio +async def test_two_problems_sharing_a_task_name_in_one_contest_are_announced( + tmp_path, +): + """Keying the model's scores by task name is what matches a human column. + + No published contest has two problems under one name (0 of 72), so this + cannot silently halve a contest without saying so. + """ + from loguru import logger + + messages: list[str] = [] + sink = logger.add(lambda m: messages.append(m.record["message"]), level="WARNING") + + twin_a = _raw(problem_id="IOI-2025-contest-dup1", task_name="samename") + twin_b = _raw(problem_id="IOI-2025-contest-dup2", task_name="samename") + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "IOI-2025-contest", + [twin_a["problem_id"], twin_b["problem_id"]], + [{"samename": 10}], + ) + ], + ) + judgement, _ = await _grade(twin_a, ["int main(){}"], [[True, True, True]]) + task = _task(twin_a) + task.dataset._contestants_path = str(contestants) + try: + await task.report( + [_Final(judgement, raw=twin_a), _Final(judgement, raw=twin_b)], [] + ) + finally: + await task.shutdown() + logger.remove(sink) + + assert any("distinct task name" in m for m in messages), messages + + +@pytest.mark.anyio +async def test_inference_with_no_rollouts_names_the_sample(): + """`next(...)` over no rollouts would raise StopIteration out of a coroutine, + which Python re-raises naming neither the sample nor the cause.""" + task = _task(_raw()) + try: + with pytest.raises(NonRetriableSampleError, match="no rollouts to grade"): + await task.feedback( + build_prediction_record([]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + + @pytest.mark.anyio async def test_the_task_is_registered_under_its_file_name(): from sieval.core.tasks.meta import get_task_class diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index c96cbd19..4e14ecc7 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -514,6 +514,18 @@ Two kinds, and the difference is a decision rather than a status: it; renaming it would widen the diff against upstream without changing the wire format. + **`test_dir` is a trusted-caller field, and the only one on this service.** + It names a host directory for the evaluator to list and read, so a client that + can reach `/evaluations` can point it anywhere the process can read. It is not + a read primitive — only `{name}.in` / `{name}.out` are opened, contents go to + the child's stdin and the comparison rather than into the response, and the + reply carries only case names and verdicts — but it is the first field that + steers the server at an arbitrary path, so it assumes the same trust boundary + the `agnostics` entry above spells out for `CODE_EVAL_AGNOSTICS_COMMAND`: + callers are in-cluster, the corpus is a mounted volume, and the service is not + exposed. A deployment that cannot hold that should run the client with + `inline_tests=True` and reject `test_dir` at the ingress. + **Deploy `docker/Dockerfile.multipl-e`** — there is no separate image for this source. The base image has no toolchain, and a `-static` link additionally needs `libstdc++-*-dev` and `libc6-dev`, but `apt-get install g++` already From 075a8e5533a92bdd0f787450e643280c365aa5a1 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 11 Sep 2026 19:42:35 +0800 Subject: [PATCH 5/5] docs(tasks): trim LiveOIBench's prose to what a reader needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review pass left its own argument in the code. Facts stay, the arguing goes — it belongs in the commit messages that made the change, not in a docstring a future reader has to walk past. `reference_impl.notes` loses the `experimental` case-making (which upstream directories are absent, how quotebench anchored instead) and keeps the two things that decide anything: no published number has been reproduced, and 380-vs-403 is why the paper's table is not the anchor. The `status` comment drops its agieval citation, `_cutoff` and `_human_metrics` lose a restated clause each, and the NULL-rubric, task-name-collision, empty-rollout and `test_dir` notes each say once what they had been saying twice. Nothing load-bearing is dropped: the scope caveat, every divergence from upstream, and the measured counts a future reader would re-verify against are all still there. 41 lines in, 65 out. Verified: 88 LiveOIBench tests and the full 7417 still pass, ruff, ty and preflight (25/25) clean, `meta/index.json` regenerated for the notes edit. Co-Authored-By: Claude Opus 5 (1M context) --- sieval/datasets/liveoibench.py | 22 +++++------ sieval/meta/index.json | 2 +- sieval/tasks/liveoibench_0shot_gen.py | 56 ++++++++++----------------- vendor/code-evaluator/VENDORED.md | 18 ++++----- vendor/code-evaluator/app/server.py | 8 ++-- 5 files changed, 41 insertions(+), 65 deletions(-) diff --git a/sieval/datasets/liveoibench.py b/sieval/datasets/liveoibench.py index 196f12da..8163e792 100644 --- a/sieval/datasets/liveoibench.py +++ b/sieval/datasets/liveoibench.py @@ -87,11 +87,10 @@ def problem_tests_dir(tests_root: str, problem_id: str) -> str: class LiveOIBenchDatasetSample(TypedDict): - # Every column the problems parquet carries, because rows are passed through - # whole (`row | {...}` below) rather than projected -- including the three - # this task never reads. `setup_script` / `evaluation_script` are non-empty - # only on the 5 script-judged problems, all of which are `interactive` and so - # are already filtered out by the default `task_type`. + # Every column the parquet carries, since rows pass through whole + # (`row | {...}`) rather than projected -- including the three never read. + # `setup_script` / `evaluation_script` are non-empty only on the 5 + # script-judged problems, all interactive and so already filtered out. id: int problem_id: str competition: str @@ -252,9 +251,9 @@ def _read_subtasks(tests_repo: str) -> dict[str, str]: between a 20 KB read and the whole corpus. """ subtasks: dict[str, str] = {} - # Tracked separately from `subtasks` being non-empty: those are two different - # failures. No file is "the corpus was never downloaded"; a file whose rubrics - # are all absent is a join failure the loader below reports per problem. + # Not the same test as `subtasks` being non-empty: no file means the corpus + # was never downloaded, where a file with no rubrics is a join failure the + # loader reports per problem. found_a_parquet = False for year in TEST_YEARS: path = os.path.join(tests_repo, year_parquet_name(year)) @@ -267,10 +266,9 @@ def _read_subtasks(tests_repo: str) -> dict[str, str]: table.column("subtasks").to_pylist(), strict=True, ): - # NOT `payload or "{}"`: a NULL cell would then read as a present but - # empty rubric, pass the loader's `is None` check below, and only - # surface per sample at grading time. Keep the absence, so the one - # place that decides what a missing rubric means is the loader. + # NOT `payload or "{}"`, which would turn a NULL cell into a present + # but empty rubric, passing the loader's own check below and only + # surfacing per sample at grading time. if payload is not None: subtasks[problem_id] = payload if not found_a_parquet: diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 57cccaed..63ea41e8 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -2410,7 +2410,7 @@ "reference_impl": { "source": "liveoibench", "url": "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/base_judge.py", - "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Aggregation follows upstream's overall block: relative_score is meaned within a contest and then across contests, pass_rate is the fraction of problems fully solved, and tests_passed_pct the fraction of test cases. Human ranking resolves a contest through the contestant table's own 'problems' column, which is where USACO's division split (-platinum / -combined) lives; all 22 USACO rows publish medal cutoffs but no contestant list, so they yield a medal and no percentile, and -combined yields neither -- upstream scores it from promotion thresholds the published dataset does not carry. Three IATI problems (senior-game, senior-ones, junior-twins) link against a grader whose header the prompt never shows, so they cannot compile and score 0; upstream behaves identically. Codeforces Elo is not computed. No published number has been reproduced, which is what keeps this 'experimental': the paper's table is over 403 problems where this scores 380, so it is not a reachable anchor, and upstream publishes no model outputs to grade against instead (no submission_results/, no results CSV, no solutions repo) -- so unlike quotebench there is no zero-cost grader anchor either. Promote to 'stable' once an n=8 run is aligned and the alignment card is filled in." + "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Aggregation follows upstream's overall block: relative_score is meaned within a contest and then across contests, pass_rate is the fraction of problems fully solved, and tests_passed_pct the fraction of test cases. Human ranking resolves a contest through the contestant table's own 'problems' column, which is where USACO's division split (-platinum / -combined) lives; all 22 USACO rows publish medal cutoffs but no contestant list, so they yield a medal and no percentile, and -combined yields neither -- upstream scores it from promotion thresholds the published dataset does not carry. Three IATI problems (senior-game, senior-ones, junior-twins) link against a grader whose header the prompt never shows, so they cannot compile and score 0; upstream behaves identically. Codeforces Elo is not computed. 'experimental' because no published number has been reproduced: the paper's table is over 403 problems where this scores 380, and upstream publishes no model outputs to anchor the grader against instead. Promote once an n=8 run is aligned." }, "status": "experimental", "reference_kind": "procedure" diff --git a/sieval/tasks/liveoibench_0shot_gen.py b/sieval/tasks/liveoibench_0shot_gen.py index fc25adea..17c028dc 100644 --- a/sieval/tasks/liveoibench_0shot_gen.py +++ b/sieval/tasks/liveoibench_0shot_gen.py @@ -75,10 +75,9 @@ def _cutoff(value) -> float | None: """A medal cutoff cell as a float, or ``None`` when the contest publishes none. - Upstream spells this ``float(x) if not pd.isna(x) else None``. The distinction - matters to :func:`medal_from_cutoffs`: a NaN that survives as a float reads as - a published cutoff nothing can clear, so the contest reports the medal - ``"None"`` and joins the medal denominator instead of staying out of it. + Upstream's ``float(x) if not pd.isna(x) else None``. A NaN surviving as a + float would read as a cutoff nothing can clear, putting the contest into the + medal denominator with the medal ``"None"`` instead of leaving it out. """ if value is None: return None @@ -101,9 +100,7 @@ def _cutoff(value) -> float | None: tags=("english", "cpp", "code-exec"), model_type="chat", reference_kind="procedure", - # Faithful port, anchor not yet reached -- the in-tree meaning of - # "experimental" (cf. agieval, which is validated on a full 7,272-row run and - # stays experimental because its anchor is unservable). See `notes` below. + # Faithful port, no anchored reproduction yet; see `notes`. status="experimental", reference_impl=ReferenceImpl( source="liveoibench", @@ -134,13 +131,10 @@ def _cutoff(value) -> float | None: "(senior-game, senior-ones, junior-twins) link against a grader whose " "header the prompt never shows, so they cannot compile and score 0; " "upstream behaves identically. Codeforces Elo is not computed. " - "No published number has been reproduced, which is what keeps this " - "'experimental': the paper's table is over 403 problems where this " - "scores 380, so it is not a reachable anchor, and upstream publishes " - "no model outputs to grade against instead (no submission_results/, " - "no results CSV, no solutions repo) -- so unlike quotebench there is " - "no zero-cost grader anchor either. Promote to 'stable' once an n=8 " - "run is aligned and the alignment card is filled in." + "'experimental' because no published number has been reproduced: the " + "paper's table is over 403 problems where this scores 380, and " + "upstream publishes no model outputs to anchor the grader against " + "instead. Promote once an n=8 run is aligned." ), ), ) @@ -234,11 +228,8 @@ async def feedback(self, post, ctx): timeout = self._request_timeout(raw, n_cases) if not post["rollouts"]: - # No rollout to pick a best from. Saying so by name is the point: - # the `next(...)` below would otherwise raise StopIteration out of a - # coroutine, which Python re-raises as a bare "coroutine raised - # StopIteration" naming neither the sample nor the cause -- the same - # failure shape the evaluator's empty-suite guard exists to avoid. + # Without this the `next(...)` below raises StopIteration out of a + # coroutine, which Python re-raises naming neither sample nor cause. raise NonRetriableSampleError( f"{raw['problem_id']}: inference returned no rollouts to grade" ) @@ -542,9 +533,8 @@ def place(ctx, result) -> None: result["metrics"]["tests_passed_pct"] if result else 0.0 ), "solved": bool(result["metrics"]["ace"]) if result else False, - # Whether this problem produced a judgement at all. The - # score-based means above take it at zero; the human - # comparison must not, see `_human_metrics`. + # The score means take a failure at zero; the human + # comparison must not -- see `_human_metrics`. "scored": result is not None, } ) @@ -573,13 +563,10 @@ def _human_metrics(self, by_contest, contestants) -> dict[str, float]: averages its per-contest percentiles. "Scored on" is why a **failed** sample is dropped here while it still - counts at zero in `relative_score` and `pass_rate`. Feeding it in would - penalise twice over: the model's total loses those points *and* the human - totals gain the column for a problem the model was never graded on. A - contest whose problems all failed therefore reports no percentile rather - than a percentile of zero, which is the honest reading -- nothing was - measured. Upstream reaches the same place by never writing a problem - result for it. + counts at zero in ``relative_score`` and ``pass_rate``: feeding it in + would penalise twice, once on the model's total and once by adding the + human column for a problem that was never graded. A contest whose + problems all failed therefore reports no percentile at all. The two counts are always reported once a contestant table was read, so ``n_contests_ranked`` of 0 reads as measured rather than as missing — @@ -602,13 +589,10 @@ def _human_metrics(self, by_contest, contestants) -> dict[str, float]: continue model_scores = {row["task_name"]: row["score"] for row in scored_rows} if len(model_scores) != len(scored_rows): - # Upstream keys this by problem id and only maps to a task column - # late; keying by task name here is what lets one human column be - # matched once. Two problems sharing a name inside one contest - # would collapse into a single entry, dropping the other from both - # the model total and the matched columns. No contest in the - # published data does this (checked: 0 of 72), so this says so - # rather than scoring a quietly smaller contest. + # Two problems sharing a name inside one contest collapse into a + # single entry, dropping the other from both the model total and + # the matched columns. No published contest does this (0 of 72), + # so say so rather than score a quietly smaller contest. logger.warning( "{}: {} scored problems share only {} distinct task name(s); " "the human comparison for this contest is incomplete.", diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index 4e14ecc7..a26092b1 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -514,17 +514,13 @@ Two kinds, and the difference is a decision rather than a status: it; renaming it would widen the diff against upstream without changing the wire format. - **`test_dir` is a trusted-caller field, and the only one on this service.** - It names a host directory for the evaluator to list and read, so a client that - can reach `/evaluations` can point it anywhere the process can read. It is not - a read primitive — only `{name}.in` / `{name}.out` are opened, contents go to - the child's stdin and the comparison rather than into the response, and the - reply carries only case names and verdicts — but it is the first field that - steers the server at an arbitrary path, so it assumes the same trust boundary - the `agnostics` entry above spells out for `CODE_EVAL_AGNOSTICS_COMMAND`: - callers are in-cluster, the corpus is a mounted volume, and the service is not - exposed. A deployment that cannot hold that should run the client with - `inline_tests=True` and reject `test_dir` at the ingress. + **`test_dir` is a trusted-caller field**, the only one here that steers the + server at an arbitrary host path. Not a read primitive — only `{name}.in` / + `{name}.out` are opened and their contents reach the child's stdin rather than + the response — but it assumes the trust boundary the `agnostics` entry above + spells out: in-cluster callers, a mounted corpus, the service not exposed. A + deployment that cannot hold that should send `inline_tests=True` instead and + reject `test_dir` at the ingress. **Deploy `docker/Dockerfile.multipl-e`** — there is no separate image for this source. The base image has no toolchain, and a `-static` link additionally diff --git a/vendor/code-evaluator/app/server.py b/vendor/code-evaluator/app/server.py index 9f07a72a..33dd0adc 100644 --- a/vendor/code-evaluator/app/server.py +++ b/vendor/code-evaluator/app/server.py @@ -649,11 +649,9 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: status=ok, msg=msg, data=ResourceMetrics( - # Both CPU fields are 0.0 on this path and that is measured, not - # missing: the per-case poller samples each child's CPU *time* in - # seconds to enforce RLIMIT_CPU, and there is no meaningful way to - # render that as the whole-service percentage this field means. - # Per-case CPU seconds stay inside the runner's verdict detail. + # Both CPU fields stay 0.0 here by construction: the per-case + # poller measures each child's CPU *seconds* to enforce + # RLIMIT_CPU, which is not this field's whole-service percentage. avg_cpu_percent=stats.cpu_percent, peak_cpu_percent=stats.peak_cpu_percent, avg_memory_mb=stats.memory_mb,