diff --git a/examples/README.md b/examples/README.md index efeb5b4b..988dbc1c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,7 @@ matches what you're trying to do, copy it, edit the marked fields, and run | [infer-recipe-override.yaml](infer-recipe-override.yaml) | Pin a specific inference recipe or override engine args | | [ruler-qwen3-8b-nonthinking.yaml](ruler-qwen3-8b-nonthinking.yaml) | RULER long-context sweep — 13 subtasks × 6 lengths (4k–128k), YaRN for 64k/128k | | [ruler-qwen3-8b-thinking.yaml](ruler-qwen3-8b-thinking.yaml) | Same RULER sweep with Qwen3 thinking enabled (adds `think_budget` + reasoning parser) | +| [liveoibench.yaml](liveoibench.yaml) | LiveOIBench — olympiad C++ problems scored by subtask; needs a C++-capable evaluator and a one-time 33.5 GB test unpack | ## Hardware-indexed (reference configs) diff --git a/examples/liveoibench.yaml b/examples/liveoibench.yaml new file mode 100644 index 00000000..dcffb75c --- /dev/null +++ b/examples/liveoibench.yaml @@ -0,0 +1,82 @@ +# ------------------------------------------------------------------------------ +# LiveOIBench — informatics-olympiad problems, graded in C++ by subtask +# ------------------------------------------------------------------------------ +# 403 problems from 72 contests across 14 olympiads (2023–2025). Scoring is +# IOI-style: a subtask pays its points only when every test in it passes, and the +# report ranks the model against the contest's real human contestants. +# +# Three-step flow, and step 2 is the one people miss: +# 1. sieval dataset download liveoibench # ~33.5 GB — the official tests +# 2. python scripts/materialize_liveoibench_tests.py +# 3. sieval eval liveoibench.yaml +# +# Step 2 unpacks the test corpus into per-problem directories. The three parquet +# files each hold a single row group, so there is no cheap per-problem read out +# of them — the loader refuses to run until they are unpacked, and says so. +# Already have upstream's `$LIVEOIBENCH_ROOT/data` tree? Point at it instead: +# args: {tests_root: /path/to/LIVEOIBENCH_ROOT/data} +# +# THIS NEEDS AN EVALUATOR WITH THE C++ SOURCE. Grading compiles the submission +# with g++ and runs it under the problem's own time and memory limits, which the +# base evaluator image cannot do — build it from +# `vendor/code-evaluator/docker/Dockerfile.multipl-e` (the image that carries a +# C++ toolchain; it serves this source too) and point SIEVAL_CODE_EVAL_API at it. +# The evaluator also reads the test directory off disk, so it needs the data +# volume mounted; if it cannot see it, set `inline_tests: true` below and accept +# ~140 MB per rollout on the wire. +# +# Scope: this runs the 380 `batch` problems. The 23 interactive ones need an +# interactor process the evaluator does not run yet, so the dataset filters them +# out rather than scoring them zero — which means these numbers are NOT +# comparable to the paper's 403-problem table. Say which you ran next to a score. +# +# Edit these fields to match your setup: +# models.local-model.infer.checkpoint — your model's on-disk path +# models.local-model.infer_meta.image — swap to your own image if you need +# custom deps, or drop the block to run +# without a container +# ------------------------------------------------------------------------------ + +result_dir: ./outputs/liveoibench + +models: + local-model: + args: + concurrency_limit: 32 + temperature: 1.0 + infer: + backend: sglang + checkpoint: /path/to/your/model # EDIT ME + infer_meta: + gpu: H100-80G + image: lmsysorg/sglang:latest + +datasets: + liveoibench: + class: LiveOIBenchDataset + # `sieval dataset download liveoibench` stages the problems repo here; the + # tests and contestant repos land beside it and are found from this path. + path: "${SIEVAL_DATA_DIR}/LiveOIBench/LiveOIBench" + args: + # Narrow the sweep while you are setting up — one competition is enough to + # prove the toolchain end to end: + # competition: IOI + # year: "2025" + task_type: batch + +tasks: + liveoibench_0shot_gen: + class: LiveOIBenchZeroShotGenTask + dataset: liveoibench + model: local-model + args: + # Upstream samples 8 candidates per problem and reports the BEST one, which + # is what the published percentiles are. At n=1 the score is a different + # (lower) quantity — reproduce upstream's protocol or say you did not. + n: 8 + # Set true only if the evaluator cannot read the materialized test tree. + inline_tests: false + infer_args: + # Olympiad solutions are long, and a truncated program compiles to nothing. + # Watch `n_truncated` in report.json and raise this if it is non-zero. + max_tokens: 32768 diff --git a/pdm.lock b/pdm.lock index 9fb06e8a..dd695add 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "drop", "hle", "ifbench", "ifeval", "iheval", "math", "multi-if", "nl2sh-alfa", "ruler", "ruler-gen", "scicode", "spider", "spider2", "t-eval", "test", "wikisql"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:ee11aa0c7ca350d44ac483278e81a61f330b54a9b0d1119250001b418fac2e61" +content_hash = "sha256:2733395ff19bb311187e466cf75746088e4f331191d869ff9dedcbe5fc05bc33" [[metadata.targets]] requires_python = ">=3.12,<3.15" diff --git a/pyproject.toml b/pyproject.toml index 2d183ae6..2a2995eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,10 @@ dependencies = [ "openai>=2.6.0", "orjson>=3.11.4", "packaging>=21.0", + # Direct, not just via datasets: sieval/datasets/liveoibench.py reads the + # LiveOIBench parquets with pyarrow, projecting columns so a 33.5 GB `tests` + # column stays unread. Floor matches the one datasets 4.2 already imposes. + "pyarrow>=21.0.0", "pyyaml>=6.0.3", "tqdm>=4.67.1", "typer>=0.24.1", diff --git a/scripts/materialize_liveoibench_tests.py b/scripts/materialize_liveoibench_tests.py new file mode 100644 index 00000000..a398e143 --- /dev/null +++ b/scripts/materialize_liveoibench_tests.py @@ -0,0 +1,148 @@ +"""Unpack the LiveOIBench test corpus into per-problem test directories. + +The published test data is three parquet files (2023/2024/2025, 33.5 GB total), +each a **single row group** whose ``tests`` column holds every problem's cases as +one JSON blob per row. That layout has no cheap random access: reading one +problem's tests means decompressing the whole column chunk. So the corpus is +unpacked once, into the directory layout upstream's ``process_dataset.py`` +produces: + + {out}/{competition}/{year}/{round}/{task}/tests/{case}.in + {out}/{competition}/{year}/{round}/{task}/tests/{case}.out + +Rows stream one at a time, so peak memory is one problem's tests (~83 MB on +average) rather than the file. Expect >30 GB written and a long run; already +materialized problems are skipped unless ``--overwrite`` is given. + +Equivalent to upstream's ``process_dataset.py --skip-metadata``, minus the +problem-metadata tree that sieval reads from the parquet directly. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import argparse +import json +import os +import shutil +import sys +from pathlib import Path + +import pyarrow.parquet as pq + +ROOT = Path(__file__).resolve().parents[1] +# Import sieval from THIS checkout rather than through the editable install, +# which may point at a different worktree. +sys.path.insert(0, str(ROOT)) + +from sieval.datasets.liveoibench import ( # noqa: E402 + TEST_YEARS, + problem_tests_dir, + year_parquet_name, +) + + +def write_problem_tests(out_dir: Path, tests: dict) -> None: + """Write one problem's cases so the directory only ever exists complete. + + A 33.5 GB unpack is long enough to be interrupted — Ctrl-C, a full disk, an + OOM kill — and both the skip check above and the dataset loader's + ``require_tests`` guard read a directory's *existence* as "this problem is + materialized". Writing in place would make a half-written problem + indistinguishable from a whole one, and it would surface much later as a + subtask naming a test case that is not there. + + So the cases land in a sibling ``tests.partial`` and are renamed over the + target in one step. A leftover ``.partial`` is a crash marker, and is + discarded the next time this problem is written. + """ + staging = out_dir.with_name(out_dir.name + ".partial") + if staging.is_dir(): + shutil.rmtree(staging) + staging.mkdir(parents=True, exist_ok=True) + for name, case in tests.items(): + case = case or {} + (staging / f"{name}.in").write_text(case.get("input", ""), encoding="utf-8") + (staging / f"{name}.out").write_text(case.get("output", ""), encoding="utf-8") + if out_dir.is_dir(): + shutil.rmtree(out_dir) + os.replace(staging, out_dir) + + +def materialize_year( + parquet_path: Path, tests_root: Path, overwrite: bool, verbose: bool = True +) -> tuple[int, int]: + """Unpack one year file. Returns (problems written, problems skipped).""" + written = skipped = 0 + parquet_file = pq.ParquetFile(parquet_path) + # batch_size=1 keeps one problem's payload in memory at a time; the column + # projection drops `subtasks`, which the dataset loader reads separately. + for batch in parquet_file.iter_batches( + batch_size=1, columns=["problem_id", "tests"] + ): + for problem_id, payload in zip( + batch.column("problem_id").to_pylist(), + batch.column("tests").to_pylist(), + strict=True, + ): + out_dir = Path(problem_tests_dir(str(tests_root), problem_id)) + if out_dir.is_dir() and not overwrite: + skipped += 1 + continue + tests = json.loads(payload or "{}") + if not isinstance(tests, dict): + raise ValueError(f"Malformed tests payload for {problem_id!r}") + write_problem_tests(out_dir, tests) + written += 1 + if verbose: + print(f" {problem_id}: {len(tests)} cases") + return written, skipped + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--tests-repo", + default=os.path.join( + os.environ.get("SIEVAL_DATA_DIR", "data"), + "LiveOIBench", + "LiveOIBench_tests", + ), + help="Directory holding the staged liveoibench_testcases_v1_*.parquet files.", + ) + parser.add_argument( + "--out", + default=None, + help="Where to write test directories (default: /materialized, " + "which is what the dataset loader looks for).", + ) + parser.add_argument("--years", nargs="*", default=list(TEST_YEARS)) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args() + + tests_repo = Path(args.tests_repo) + tests_root = Path(args.out) if args.out else tests_repo / "materialized" + + total_written = total_skipped = 0 + for year in args.years: + parquet_path = tests_repo / year_parquet_name(year) + if not parquet_path.exists(): + print( + f"missing {parquet_path}; " + "run `sieval dataset download liveoibench` first" + ) + return 1 + print(f"unpacking {parquet_path.name} -> {tests_root}") + written, skipped = materialize_year( + parquet_path, tests_root, args.overwrite, verbose=not args.quiet + ) + print(f" {year}: {written} written, {skipped} already present") + total_written += written + total_skipped += skipped + + print(f"done: {total_written} problems written, {total_skipped} skipped") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sieval/community/liveoibench/__init__.py b/sieval/community/liveoibench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sieval/community/liveoibench/code_extractor.py b/sieval/community/liveoibench/code_extractor.py new file mode 100644 index 00000000..a39517ab --- /dev/null +++ b/sieval/community/liveoibench/code_extractor.py @@ -0,0 +1,208 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/code_extractor.py +""" +Utility for extracting code from LLM responses. + +This module provides a centralized way to extract and save code from +various LLM response formats. +""" + +import re +import os +import json +from typing import Optional, Tuple + + +class CodeExtractor: + """Utility class for extracting code from LLM responses.""" + + @staticmethod + def extract_code(content: str, task_name: str) -> Tuple[Optional[str], str]: + """ + Extract code from LLM response content using prioritized patterns. + + Args: + content: The response content from the LLM + task_name: Name of the task (used for pattern matching) + + Returns: + Tuple of (extracted_code, status) where status is one of: + 'success', 'empty', 'not_found' + """ + if not content: + return None, 'not_found' + + # Prioritized patterns for code extraction + patterns = [ + # 1) ```{task}.cpp or ```{task}.c ... ``` + rf"```(?:{re.escape(task_name)}\.(?:cpp|c))\s*([\s\S]*?)```", + # 2) ```cpp or ```c ... ``` + r"```(?:cpp|c)\s*([\s\S]*?)```", + # 3) Generic ``` ... ``` + r"```([\s\S]*?)```", + ] + + code = None + for pattern in patterns: + matches = re.findall(pattern, content, re.DOTALL) + if matches: + # Get the longest code block + code = max(matches, key=lambda x: len(x.split('\n'))).strip() + break + + if code is None: + return None, 'not_found' + + if not code: + return "", 'empty' + + # Post-process the code + code = CodeExtractor._post_process_code(code) + return code, 'success' + + @staticmethod + def _post_process_code(code: str) -> str: + """ + Post-process extracted code to fix common issues. + + Args: + code: Raw extracted code + + Returns: + Cleaned code + """ + lines = code.split('\n') + + # Check if the code is too short + if len(lines) < 5: + # This might be an issue but we'll keep it + pass + + # Fix first line if it contains just "cpp" or ends with .cpp + if lines: + first_line = lines[0].strip() + if first_line.lower() == "cpp" or first_line.endswith(".cpp"): + lines[0] = "// " + first_line + + return '\n'.join(lines) + + @staticmethod + def save_code( + code: Optional[str], + prediction_path: str, + task: str, + model: str, + seed: int + ): + """ + Save extracted code to a file. + + Args: + code: Extracted code (can be None or empty) + prediction_path: Path to save predictions + task: Task name + model: Model name + seed: Random seed number + """ + # Create codes directory if it doesn't exist + os.makedirs(os.path.join(prediction_path, "codes"), exist_ok=True) + + # Save code (even if empty) + code_path = os.path.join(prediction_path, "codes", f"{task}_{model}_{seed}.cpp") + with open(code_path, "w") as f: + f.write(code if code else "") + + @staticmethod + def save_raw_response( + response: any, + prediction_path: str, + task: str, + model: str, + seed: int + ): + """ + Save raw API response to a JSON file. + + Args: + response: API response object + prediction_path: Path to save predictions + task: Task name + model: Model name + seed: Random seed number + """ + if response is None: + return + + # Skip empty responses + if hasattr(response, "choices") and hasattr(response, "usage"): + if response.usage.completion_tokens == 0: + return + + # Create raw directory if it doesn't exist + os.makedirs(os.path.join(prediction_path, "raw"), exist_ok=True) + + # Save raw response + raw_path = os.path.join(prediction_path, "raw", f"{task}_{model}_{seed}.json") + + # Handle different response formats + try: + if hasattr(response, "model_dump_json"): + with open(raw_path, "w") as f: + f.write(response.model_dump_json()) + elif hasattr(response, "to_dict"): + with open(raw_path, "w") as f: + json.dump(response.to_dict(), f) + else: + with open(raw_path, "w") as f: + json.dump(response, f) + except Exception as e: + print(f"Warning: Could not save raw response for {task} seed {seed}: {e}") + + @staticmethod + def extract_from_response(response: any) -> Optional[str]: + """ + Extract content from an API response object. + + Args: + response: API response object + + Returns: + Response content as string, or None if not available + """ + if response is None: + return None + + if not hasattr(response, "choices") or len(response.choices) == 0: + return None + + if hasattr(response.choices[0], "message") and hasattr(response.choices[0].message, "content"): + return response.choices[0].message.content + + return None + + +class ExtractionStats: + """Track code extraction statistics.""" + + def __init__(self): + self.success = 0 + self.failed = 0 + self.empty = 0 + + def record(self, status: str): + """Record an extraction result.""" + if status == 'success': + self.success += 1 + elif status == 'empty': + self.empty += 1 + else: + self.failed += 1 + + def print_summary(self): + """Print extraction statistics summary.""" + print("\nCode Extraction Statistics:") + print(f" Successful extractions: {self.success}") + print(f" Failed extractions: {self.failed}") + print(f" Empty code blocks: {self.empty}") + + def __str__(self): + return f"Success: {self.success}, Failed: {self.failed}, Empty: {self.empty}" diff --git a/sieval/community/liveoibench/payloads.py b/sieval/community/liveoibench/payloads.py new file mode 100644 index 00000000..1d706b08 --- /dev/null +++ b/sieval/community/liveoibench/payloads.py @@ -0,0 +1,43 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/process_dataset.py +# (_normalize_code_mapping, and the subtasks parse in write_problem_assets) +"""Parsing the JSON-in-a-column payloads the LiveOIBench parquets carry. + +Divergence from upstream: upstream decodes these while writing files to disk and +raises ``DatasetError`` on a malformed subtasks blob; here the same decoding +happens at load time. A malformed *code* bundle still reads as "this problem +ships none", which is upstream's behaviour and the common case — 362 of 403 +problems have no grader bundle at all. +""" + +import json +import math + + +def load_subtasks(raw_subtasks: str | None) -> dict[str, dict]: + """Parse the ``subtasks`` column into upstream's ``{id: subtask}`` mapping.""" + parsed = json.loads(raw_subtasks or "{}") + if not isinstance(parsed, dict): + raise ValueError(f"Malformed subtasks payload: {type(parsed).__name__}") + return parsed + + +def load_code_bundle(raw_value: object) -> dict[str, str]: + """Parse a ``grader_code`` / ``starter_code`` cell into filename → contents.""" + if raw_value is None: + return {} + if isinstance(raw_value, float) and math.isnan(raw_value): + return {} + if isinstance(raw_value, bytes): + raw_value = raw_value.decode("utf-8") + if not isinstance(raw_value, str): + return {} + stripped = raw_value.strip() + if not stripped: + return {} + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return {} + if not isinstance(parsed, dict): + return {} + return {str(name): str(contents) for name, contents in parsed.items()} diff --git a/sieval/community/liveoibench/prompts.py b/sieval/community/liveoibench/prompts.py new file mode 100644 index 00000000..010cb4e2 --- /dev/null +++ b/sieval/community/liveoibench/prompts.py @@ -0,0 +1,74 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/problem.py +# (Problem.get_prompt) and .../src/process_dataset.py (ParquetReconstructor._write_prompt), +# which build the same string from the same fields — one from a reconstructed +# problem directory, the other from the parquet row. +"""Prompt construction for LiveOIBench. + +Divergence from upstream: upstream reads ``statements/statement.md`` and the +``attachments/`` directory off disk, after ``process_dataset.py`` has +materialized them. Here the same values arrive as the parquet fields they were +written from — ``problem_statement`` and the ``starter_code`` bundle (a JSON +map of filename to contents) — so the file-existence tests become key lookups. +The assembled string is unchanged, including the ``time_limit`` / +``memory_limit`` floats rendering as ``1.5`` / ``2048.0``. + +Upstream's grader block requires all three of ``grader.cpp`` (or ``stub.cpp``), +``{task}.h`` and ``{task}.cpp``; 17 of the 403 published problems satisfy that, +and the rest take the plain branch. +""" + +from typing import Mapping + + +def build_prompt( + task: str, + statement: str, + time_limit: float | None, + memory_limit: float | None, + starter_codes: Mapping[str, str] | None = None, +) -> str: + """Return the user prompt for one problem. + + *task* is upstream's task token — the ``problem_id`` remainder after + competition/year/round, which the dataset carries as ``task_name``. It names + the fence the model is asked to use, so it is also what + ``CodeExtractor.extract_code`` matches on. + """ + starter_codes = starter_codes or {} + + prompt = ( + f"Given a competition problem below, write a solution in C++ that solves all the subtasks. " + f"Make sure to wrap your code in '```{task}.cpp' and '```' Markdown delimiters.\n\n" + ) + if statement: + prompt += "[BEGIN PROBLEM]\n" + prompt += statement + prompt += "[END PROBLEM]\n" + + time_text = time_limit if time_limit is not None else "unknown" + memory_text = memory_limit if memory_limit is not None else "unknown" + prompt += f"Time limit: {time_text} seconds\n" + prompt += f"Memory limit: {memory_text} MB\n" + + grader_code = starter_codes.get("grader.cpp") + if grader_code is None: + grader_code = starter_codes.get("stub.cpp") + header_code = starter_codes.get(f"{task}.h") + solution_code = starter_codes.get(f"{task}.cpp") + + if grader_code is not None and header_code is not None and solution_code is not None: + prompt += ( + "We are going to grade your solution using the following grader.cpp file and " + f"{task}.h file. You can write your solution by modifying the {task}.cpp and wrap " + f"your code in '```{task}.cpp' and '```' Markdown delimiters.\n\n" + ) + prompt += "```grader.cpp\n" + grader_code + "```\n\n" + prompt += f"```{task}.h\n" + header_code + "```\n\n" + prompt += f"```{task}.cpp\n" + solution_code + "```\n\n" + else: + prompt += ( + f"Generate a solution in C++ that solves the task. Make sure to wrap your code in " + f"'```{task}.cpp' and '```' Markdown delimiters.\n\n" + ) + + return prompt diff --git a/sieval/community/liveoibench/rankings.py b/sieval/community/liveoibench/rankings.py new file mode 100644 index 00000000..bd909d49 --- /dev/null +++ b/sieval/community/liveoibench/rankings.py @@ -0,0 +1,371 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/generate_rankings.py +# (normalize_name, normalize_contest_identifier, NON_TASK_COLUMNS, +# identify_task_columns, calculate_percentile, build_problem_to_contest_map, +# the contest-routing block of group_problems_by_contest, and compute_human_metrics) +"""Ranking a model against the contest's human contestants. + +Divergence from upstream: + +* Upstream operates on a pandas DataFrame built from ``contestants_ranking``; + here the rows arrive as the list of dicts that column actually holds, so + ``df.columns`` becomes the union of row keys and the vectorized column sum + becomes a per-row sum. The numbers are the same. +* Codeforces Elo is not ported — it is a separate upstream fit against + contestant CF ratings. +* Upstream's USACO ``-combined`` contests are scored from promotion thresholds + in ``data/USACO/{year}/{contest}/contest_info.json``, a file that lives in the + upstream repository rather than in the published dataset. Without it upstream + itself returns no medal and no percentile for those contests, which is what + :func:`score_contest` returns; see the ``-combined`` branch there. + +Contest identity is **not** derivable from the problem id alone. USACO keys its +contestant rows by division (``USACO-2023-December_Contest-platinum`` / +``-combined``), so :func:`build_problem_to_contest_map` reads the authoritative +mapping out of the contestant table's own ``problems`` column and +:func:`resolve_contest_id` falls back to upstream's division ladder. On the +published data every problem is listed by exactly one contest row. + +Human totals are re-summed over **only** the tasks the model was scored on, so a +contest whose problems were partly filtered out (interactive ones are) still +compares like with like. Medal cutoffs are the contest's published ones and are +not rescaled, which is upstream's behaviour. +""" + +import json +import re +from typing import Any, Iterable, Mapping, Sequence + +NON_TASK_COLUMNS = { + "rank", + "contestant", + "country", + "total", + "recalculated_total", + "medal", + "cf_rating", + "day1", + "day2", + "day 1", + "day 2", + "score rel.", + "division", + "team", + "nationality", +} + +_FALLBACK_TOTAL_COLUMNS = ("Total", "total", "Total Score", "score", "Score") +_RECALC_TOTAL_COLUMNS = ("Recalculated_Total", "recalculated_total") +_RANK_COLUMNS = ("Rank", "rank") +_PROMOTED_DIVISIONS = {"bronze", "silver", "gold"} + +# Upstream renames the three Canadian rounds when matching the human leaderboard. +_CCO_ROUND_ALIASES = { + "Canadian_Computing_Competition_Junior": "Junior", + "Canadian_Computing_Competition_Senior": "Senior", + "Canadian_Computing_Olympiad": "contest", +} + + +def normalize_name(value: str) -> str: + """Normalize task column names for consistent matching.""" + return re.sub(r"[^a-z0-9]", "", value.lower()) + + +def normalize_contest_identifier(contest_base: str) -> str: + """Upstream ``normalize_contest_identifier``: rename the CCO rounds. + + The published contestant table happens to key CCO by its *un*-renamed round, + which is why :func:`resolve_contest_id` keeps upstream's fallback to the raw + identifier. Ported so the ladder is the same one upstream walks. + """ + if not contest_base.startswith("CCO-"): + return contest_base + parts = contest_base.split("-", 2) + if len(parts) != 3: + return contest_base + prefix, year, rest = parts + for source, replacement in _CCO_ROUND_ALIASES.items(): + if rest.startswith(source): + rest = rest.replace(source, replacement, 1) + break + return f"{prefix}-{year}-{rest}" + + +def usaco_division(problem_id: str) -> str | None: + """The division token a USACO problem id carries, or ``None`` elsewhere. + + ``USACO-2023-US_Open_Contest-bronze_FEB`` -> ``"bronze"``. Upstream reads it + off the same fourth ``-`` segment, which is also the ``task_name`` column. + """ + if not problem_id.startswith("USACO-"): + return None + parts = problem_id.split("-", 3) + if len(parts) < 4: + return None + return parts[3].split("_", 1)[0].lower() + + +def build_problem_to_contest_map( + contest_rows: Iterable[Mapping[str, Any]], +) -> dict[str, str]: + """Upstream ``build_problem_to_contest_map``: problem id -> contest id. + + Driven by each contestant row's ``problems`` column, which is the only place + the USACO division split is stated. The division filter is upstream's and is + a no-op on consistent data — a ``-combined`` row lists only bronze/silver/gold + problems and a ``-platinum`` row only platinum ones. + """ + mapping: dict[str, str] = {} + for row in contest_rows: + contest_id = row.get("contest_id") + problems = row.get("problems") + if not contest_id or problems is None: + continue + if isinstance(problems, str): + try: + problems = json.loads(problems) + except json.JSONDecodeError: + continue + if not isinstance(problems, (list, tuple)): + continue + for problem in problems: + problem_id = str(problem) + division = usaco_division(problem_id) + if division is None: + mapping[problem_id] = contest_id + elif contest_id.endswith("-combined"): + if division in _PROMOTED_DIVISIONS: + mapping[problem_id] = contest_id + elif contest_id.endswith("-platinum"): + if division == "platinum": + mapping[problem_id] = contest_id + else: + mapping[problem_id] = contest_id + return mapping + + +def resolve_contest_id( + problem_id: str, + contest_base: str, + problem_to_contest: Mapping[str, str], + available_contests: Sequence[str] | set[str] = (), +) -> str | None: + """Which contest row one problem is ranked against; upstream's ladder. + + *contest_base* is ``{competition}-{year}-{round}``, the identifier the + problem id alone yields. ``None`` means the problem has no human ranking — + upstream's ``continue``, which drops a USACO problem whose division token is + not one of the four. + """ + available = set(available_contests) + contest_id = problem_to_contest.get(problem_id) + base_norm = normalize_contest_identifier(contest_base) + + if not contest_id: + division = usaco_division(problem_id) + if division is not None: + if division == "platinum": + contest_id = f"{base_norm}-platinum" + elif division in _PROMOTED_DIVISIONS: + contest_id = f"{base_norm}-combined" + else: + return None + else: + contest_id = base_norm + + if contest_id not in available and contest_id != base_norm and base_norm in available: + contest_id = base_norm + if contest_id not in available and contest_base in available: + contest_id = contest_base + return contest_id + + +def identify_task_columns(columns: Sequence[str], contest_id: str = "") -> dict[str, str]: + """Return a mapping from normalized task name to original column name.""" + mapping: dict[str, str] = {} + normalized_exclusions = {normalize_name(col) for col in NON_TASK_COLUMNS} + contest_lower = (contest_id or "").lower() + + # BOI scores by day, so `day1` / `day2` are tasks there rather than headers. + if "boi-" in contest_lower: + for day_col in ("day1", "day2"): + normalized_exclusions.discard(day_col) + + for column in columns: + norm = normalize_name(column) + if norm and norm not in normalized_exclusions: + mapping[norm] = column + return mapping + + +def calculate_percentile(model_score: float, human_scores: Sequence[float]) -> float | None: + """Percent of contestants the model strictly outscores.""" + if not human_scores: + return None + better = sum(1 for score in human_scores if model_score > score) + return (better / len(human_scores)) * 100 + + +def medal_from_cutoffs( + total: float, + gold_cutoff: float | None, + silver_cutoff: float | None, + bronze_cutoff: float | None, +) -> str | None: + """Upstream's cutoff ladder: ``None`` only when the contest publishes none.""" + if gold_cutoff is not None and total >= gold_cutoff: + return "Gold" + if silver_cutoff is not None and total >= silver_cutoff: + return "Silver" + if bronze_cutoff is not None and total >= bronze_cutoff: + return "Bronze" + if any(cutoff is not None for cutoff in (gold_cutoff, silver_cutoff, bronze_cutoff)): + return "None" + return None + + +def _to_numeric(value: Any) -> float: + """``pd.to_numeric(errors="coerce").fillna(0.0)`` for a single cell.""" + if value is None or isinstance(value, bool): + return 0.0 + try: + number = float(value) + except (TypeError, ValueError): + return 0.0 + return 0.0 if number != number else number # NaN -> 0.0 + + +def _numeric_or_none(value: Any) -> float | None: + """``pd.to_numeric(errors="coerce")`` for a single cell, keeping NaN as None. + + The recalculated-total branch *drops* unparseable rows rather than zeroing + them (upstream ``.dropna()``), so it needs the distinction ``_to_numeric`` + throws away. + """ + if value is None or isinstance(value, bool): + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return None if number != number else number + + +def _result( + human_percentile: float | None, + medal: str | None, + model_total: float, + matched_columns: list[str], + n_contestants: int, +) -> dict: + return { + "human_percentile": human_percentile, + "medal": medal, + "model_total": model_total, + "matched_columns": matched_columns, + "n_contestants": n_contestants, + } + + +def score_contest( + rankings: Sequence[Mapping[str, Any]], + model_scores: Mapping[str, float], + contest_id: str = "", + gold_cutoff: float | None = None, + silver_cutoff: float | None = None, + bronze_cutoff: float | None = None, +) -> dict: + """Rank one model's contest total against that contest's contestants. + + *model_scores* maps a problem's ``task_name`` to the model's score on it, for + the problems of this contest that were actually evaluated. + """ + model_sum = float(sum(model_scores.values())) + + # Upstream short-circuits a USACO `-combined` contest into + # `compute_usaco_combined_metrics`, which compares each division's total + # against the promotion thresholds in that contest's `contest_info.json`. + # That file is part of the upstream repository, not of the published + # dataset, so upstream on this data reaches `thresholds_present = False` and + # returns no medal and no percentile. Returning anything here -- the row's + # own medal cutoffs, say -- would be a number upstream does not compute. + if contest_id.startswith("USACO-") and contest_id.endswith("-combined"): + return _result(None, None, model_sum, [], len(rankings)) + + # No contestant rows: upstream's `if df.empty` branch still awards a medal + # from the published cutoffs, and only the percentile is unavailable. Every + # USACO contest in the release takes this path. + if not rankings: + medal = medal_from_cutoffs(model_sum, gold_cutoff, silver_cutoff, bronze_cutoff) + return _result(None, medal, model_sum, [], 0) + + columns: list[str] = [] + for row in rankings: + for key in row: + if key not in columns: + columns.append(key) + + # Upstream prefers a `Recalculated_Total` column wherever one exists: the + # contest's own total re-derived over the tasks LiveOIBench scores, which is + # not the same number as the sum of the per-task columns (it differs for + # 20-83% of contestants on the 10 contests that carry it). The model side + # stays the sum of every problem evaluated in the contest. + recalc_column = next((col for col in _RECALC_TOTAL_COLUMNS if col in columns), None) + if recalc_column is not None: + human_totals = [ + value + for value in (_numeric_or_none(row.get(recalc_column)) for row in rankings) + if value is not None + ] + human_percentile = calculate_percentile(model_sum, human_totals) + # On this branch only, upstream re-derives the Canadian Computing + # Olympiad percentile from the rank column. No published CCO contest + # carries `Recalculated_Total`, so this is unreachable on the released + # data; ported because dropping it would make the port wrong for a + # release that does. + if "canadian_computing_olympiad" in contest_id.lower() and any( + col in columns for col in _RANK_COLUMNS + ): + rank_column = next(col for col in _RANK_COLUMNS if col in columns) + n_ranked = sum( + 1 for row in rankings if _numeric_or_none(row.get(rank_column)) is not None + ) + if n_ranked: + model_rank = sum(1 for total in human_totals if total > model_sum) + 1 + model_rank = max(1, min(model_rank, n_ranked)) + human_percentile = ((n_ranked - model_rank) / n_ranked) * 100 + medal = medal_from_cutoffs(model_sum, gold_cutoff, silver_cutoff, bronze_cutoff) + return _result(human_percentile, medal, model_sum, [recalc_column], len(rankings)) + + task_column_map = identify_task_columns(columns, contest_id) + + matched_columns: list[str] = [] + matched_scores: list[float] = [] + for task_name, score in model_scores.items(): + column = task_column_map.get(normalize_name(str(task_name))) + if column: + matched_columns.append(column) + matched_scores.append(score) + + if matched_columns: + model_total = float(sum(matched_scores)) + else: + # No per-task columns: fall back to the contest total, against which the + # model's own total is every evaluated problem summed. + fallback = next((col for col in _FALLBACK_TOTAL_COLUMNS if col in columns), None) + if fallback is None: + return _result(None, None, model_sum, [], len(rankings)) + matched_columns = [fallback] + model_total = model_sum + + human_totals = [ + sum(_to_numeric(row.get(column)) for column in matched_columns) for row in rankings + ] + + return _result( + calculate_percentile(model_total, human_totals), + medal_from_cutoffs(model_total, gold_cutoff, silver_cutoff, bronze_cutoff), + model_total, + matched_columns, + len(rankings), + ) diff --git a/sieval/community/liveoibench/scoring.py b/sieval/community/liveoibench/scoring.py new file mode 100644 index 00000000..675df43b --- /dev/null +++ b/sieval/community/liveoibench/scoring.py @@ -0,0 +1,74 @@ +# adapted from https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/base_judge.py +# (BaseJudge.interprete_task_result) +"""Subtask scoring for LiveOIBench. + +Divergence from upstream: none in the arithmetic. Upstream is a method on the +judge that has just executed the tests; here the per-test verdicts arrive from +the evaluator service, so the same function takes them as an argument. The +misspelled name is upstream's and is kept so the two can be diffed. + +The ``min-score`` branch is reachable only with a custom checker, which supplies +a fractional per-test ``score``. No checker ships in the published dataset (see +``process_dataset.py``, which materializes no ``checkers/`` directory), so on +that data every test scores 1.0 or 0.0 and the branch collapses onto the +all-or-nothing rule below it. It is ported anyway, because dropping it would +make the port wrong for any release that does ship checkers. +""" + +from typing import Any, Mapping, Sequence + + +def interprete_task_result( + results: Sequence[Mapping[str, Any]], + subtasks: Mapping[str, Mapping[str, Any]], +) -> dict: + """Score one submission from its per-test verdicts. + + *results* is one mapping per executed test with ``test_case`` (the test's + name, no extension), ``correct`` (bool) and optionally ``score`` (float, + checker-supplied). *subtasks* is the problem's ``subtasks`` payload: an id + to ``{"score": int, "testcases": [name, ...]}`` mapping, optionally with + ``"grading": "min-score"``. + + Returns ``{"subtasks": {id: {...}}, "score": int, "tests_passed": float, + "ace": bool}``. + """ + score: dict[str, Any] = {"subtasks": {}} + results_dict = {result["test_case"]: result for result in results} + test_cases_passed = [result["test_case"] for result in results if result["correct"]] + + for i, subtask in subtasks.items(): + subtask_score = 0 + subtask_passed = 0 + + for test_case in subtask["testcases"]: + if results_dict[test_case]["correct"]: + subtask_passed += 1 + + # Handle min-score grading + if "grading" in subtask and subtask["grading"] == "min-score": + test_scores = [ + results_dict[test_case].get("score", 0) for test_case in subtask["testcases"] + ] + subtask_score = subtask["score"] * min(test_scores) + + # Full score if all tests passed + if subtask_passed == len(subtask["testcases"]): + subtask_score = subtask["score"] + + score["subtasks"][i] = { + "testcases": len(subtask["testcases"]), + "score": subtask_score, + "passed": subtask_passed / len(subtask["testcases"]), + } + + score["score"] = round(sum([subtask["score"] for subtask in score["subtasks"].values()])) + score["tests_passed"] = len(test_cases_passed) / len(results) if results else 0 + score["ace"] = score["tests_passed"] == 1 + + return score + + +def total_points(subtasks: Mapping[str, Mapping[str, Any]]) -> int: + """The problem's maximum score — upstream ``Problem.get_total_points``.""" + return sum(int(subtask["score"]) for subtask in subtasks.values()) diff --git a/sieval/datasets/__init__.pyi b/sieval/datasets/__init__.pyi index fa8d0d92..b5c5fb8c 100644 --- a/sieval/datasets/__init__.pyi +++ b/sieval/datasets/__init__.pyi @@ -141,6 +141,10 @@ from .livecodebench_code_generation import ( LiveCodeBenchDataset, LiveCodeBenchDatasetSample, ) +from .liveoibench import ( + LiveOIBenchDataset, + LiveOIBenchDatasetSample, +) from .math_500 import ( MATH500Dataset, MATH500DatasetSample, @@ -317,6 +321,8 @@ __all__ = [ "InverseIFEvalDatasetSample", "LiveCodeBenchDataset", "LiveCodeBenchDatasetSample", + "LiveOIBenchDataset", + "LiveOIBenchDatasetSample", "MATH500Dataset", "MATH500DatasetSample", "MATHPerturbHardDataset", diff --git a/sieval/datasets/liveoibench.py b/sieval/datasets/liveoibench.py new file mode 100644 index 00000000..8163e792 --- /dev/null +++ b/sieval/datasets/liveoibench.py @@ -0,0 +1,279 @@ +"""LiveOIBench dataset — informatics-olympiad problems with official test data. + +403 problems from 72 contests across 14 olympiads (2023–2025), published as three +HuggingFace repos that this dataset stages together: + +* problems and metadata (920 KB) — statements, limits, grader/starter bundles; +* contestant results (559 KB) — per-contest human rankings and medal cutoffs, + kept on :attr:`contestants_path` for the task's report to rank against; +* official test cases (**33.5 GB**) — one parquet per year, each a single row + group, so there is no cheap per-problem read out of them. + +Because of that last point the test corpus is **materialized once** into +per-problem directories — the layout upstream's ``process_dataset.py`` produces, +so an existing ``$LIVEOIBENCH_ROOT/data`` tree can be pointed at directly. Rows +carry the path rather than the payload: at ~83 MB of tests per problem, a column +holding them would put the whole corpus through the record pipeline. + +The ``subtasks`` column is joined in from the test parquets, where it lives; it +is ~20 KB per year, so column projection reads it without touching ``tests``. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import os +from typing import TypedDict, override + +import pyarrow.parquet as pq +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.datasets import ( + Category, + Dataset, + Level1Category, + sieval_dataset, +) +from sieval.core.utils.hf import ensure_dataset_dict + +# `load_subtasks` / `load_code_bundle` are ports of upstream's own payload +# decoding and live in community/ next to the rest of it; the task reads the same +# two functions from there rather than reaching into this module. + +LIVEOIBENCH_REVISION = "0e87d7c7a667979e00df41889656d73f1d3ce153" +LIVEOIBENCH_TESTS_REVISION = "05731838fc460d3235c70506ee177aeb68a610cf" +LIVEOIBENCH_CONTESTANTS_REVISION = "fdcc2ba59670b06ec55ee61bc771ed855ba6dba6" + +PROBLEMS_FILE = os.path.join("data", "liveoibench_v1.parquet") +CONTESTANTS_FILE = os.path.join("data", "contest_results.parquet") +TEST_YEARS = ("2023", "2024", "2025") + + +def year_parquet_name(year: str) -> str: + """The test parquet for one year. + + Deliberately not spelled ``test_*``: pytest collects any imported callable + with that prefix as a test case, so a test module using this helper would + fail to collect on the helper itself. + """ + return f"liveoibench_testcases_v1_{year}.parquet" + + +def parse_problem_id(problem_id: str) -> tuple[str, str, str, str]: + """Split ``competition-year-round-task``; upstream ``parse_problem_id``. + + The task token is everything after the third segment, so a task name that + itself contains ``-`` (``grading-server``) survives. It is the same value the + ``task_name`` column carries, and it is what the prompt's code fence and the + extractor's first pattern are built from. + """ + parts = problem_id.split("-") + if len(parts) < 4: + raise ValueError( + f"Invalid problem_id {problem_id!r}; expected at least 4 tokens." + ) + return parts[0], parts[1], parts[2], "-".join(parts[3:]) + + +def problem_tests_dir(tests_root: str, problem_id: str) -> str: + """Where one problem's ``.in`` / ``.out`` files live once materialized. + + Upstream's reconstructed layout + (``{competition}/{year}/{round}/{task}/tests``), so a tree already built by + ``process_dataset.py`` can be used as *tests_root* unchanged. + """ + competition, year, round_name, task = parse_problem_id(problem_id) + return os.path.join(tests_root, competition, year, round_name, task, "tests") + + +class LiveOIBenchDatasetSample(TypedDict): + # Every column the parquet carries, since rows pass through whole + # (`row | {...}`) rather than projected -- including the three never read. + # `setup_script` / `evaluation_script` are non-empty only on the 5 + # script-judged problems, all interactive and so already filtered out. + id: int + problem_id: str + competition: str + contest: str + task_name: str + setup_script: str + evaluation_script: str + problem_statement: str + time_limit: float + memory_limit: float + task_type: str + difficulty: int + algorithms: str + grader_code: str + starter_code: str + # Joined from the test parquet: `{id: {"score": int, "testcases": [name]}}`. + subtasks: str + # Derived. `contest_id` is the join key into the contestant table, whose own + # column is spelled that way; `tests_dir` is where this problem's test files + # were materialized. + contest_id: str + tests_dir: str + + +@sieval_dataset( + name="liveoibench", + display_name="LiveOIBench", + description=( + "403 olympiad problems with official tests, subtask rubrics and human " + "rankings; 33.5 GB of tests." + ), + source=( + f"hf:LiveOIBench/LiveOIBench@{LIVEOIBENCH_REVISION}", + f"hf:LiveOIBench/LiveOIBench_tests@{LIVEOIBENCH_TESTS_REVISION}", + f"hf:LiveOIBench/LiveOIBench_contestants@{LIVEOIBENCH_CONTESTANTS_REVISION}", + ), + categories=(Category(Level1Category.CODE, "CodeGeneration"),), + tags=("english", "cpp", "code-exec"), + license="cc-by-4.0", +) +class LiveOIBenchDataset(Dataset[LiveOIBenchDatasetSample]): + @override + def load( + self, + name_or_path: str, + task_type: str | None = "batch", + year: str | None = None, + competition: str | None = None, + tests_root: str | None = None, + require_tests: bool = True, + **kwargs, + ) -> HFDatasetDict: + """Load the problems, joined with their subtasks and test locations. + + *task_type* defaults to ``"batch"`` — the 380 problems the evaluator's + C++ path can grade. The 23 ``interactive`` ones need an interactor + process it does not run yet, and are filtered out rather than scored + zero, which would understate every model invisibly. Pass ``None`` to keep + every problem. + + *tests_root* defaults to a ``tests/`` tree beside the staged parquet + files; point it at an existing ``$LIVEOIBENCH_ROOT/data`` to reuse one. + *require_tests* is the guard that turns a missing corpus into an + actionable error instead of a run that fails 380 times. + """ + staged_root = os.path.dirname(os.path.normpath(name_or_path)) + problems_path = os.path.join(name_or_path, PROBLEMS_FILE) + tests_repo = os.path.join(staged_root, "LiveOIBench_tests") + contestants_repo = os.path.join(staged_root, "LiveOIBench_contestants") + + # Captured for the task, the way SciCode captures its h5 path; + # copy.copy-based clones (slice/shuffle/...) preserve the attribute. + self._contestants_path = os.path.join(contestants_repo, CONTESTANTS_FILE) + self._tests_root = tests_root or os.path.join(tests_repo, "materialized") + + table = pq.read_table(problems_path) + rows: list[dict] = table.to_pylist() + + subtasks_by_id = _read_subtasks(tests_repo) + kept: list[dict] = [] + for row in rows: + problem_id = row["problem_id"] + if task_type is not None and row["task_type"] != task_type: + continue + competition_name, problem_year, round_name, _task = parse_problem_id( + problem_id + ) + if year is not None and problem_year != year: + continue + if competition is not None and competition_name != competition: + continue + subtasks = subtasks_by_id.get(problem_id) + if subtasks is None: + # A problem with no subtask rubric has no score to compute; that + # is a broken join, not an empty result. + raise ValueError( + f"No subtasks found for {problem_id!r} in {tests_repo!r}; " + "the problem and test parquets are out of step." + ) + kept.append( + row + | { + "subtasks": subtasks, + "contest_id": "-".join( + (competition_name, problem_year, round_name) + ), + "tests_dir": problem_tests_dir(self._tests_root, problem_id), + } + ) + + if not kept: + raise ValueError( + "LiveOIBench dataset is empty after filtering " + f"(task_type={task_type!r}, year={year!r}, " + f"competition={competition!r})." + ) + + if require_tests: + missing = [ + row["problem_id"] for row in kept if not os.path.isdir(row["tests_dir"]) + ] + if missing: + raise FileNotFoundError( + f"{len(missing)} of {len(kept)} problems have no materialized " + f"test directory under {self._tests_root!r} " + f"(e.g. {missing[0]!r}). The 33.5 GB test corpus ships as " + "three single-row-group parquets and has to be unpacked " + "once:\n" + " python scripts/materialize_liveoibench_tests.py\n" + "Pass tests_root= to reuse a tree built by upstream's " + "process_dataset.py, or require_tests=False to load prompts " + "without grading." + ) + + return ensure_dataset_dict(HFDatasetDict({"test": HFDataset.from_list(kept)})) + + @property + def contestants_path(self) -> str | None: + """Staged path of the contestant-results parquet, for ranking a model. + + ``None`` when the dataset was built from a pre-loaded dict rather than + via :meth:`load`. + """ + return getattr(self, "_contestants_path", None) + + @property + def tests_root(self) -> str | None: + """Root of the materialized test tree; ``None`` as for + :attr:`contestants_path`.""" + return getattr(self, "_tests_root", None) + + +def _read_subtasks(tests_repo: str) -> dict[str, str]: + """Read every year's ``subtasks`` column, skipping the ``tests`` column. + + Each year file is one row group whose ``tests`` column is 6–17 GB and whose + ``subtasks`` column is ~20 KB, so projecting to two columns is the difference + between a 20 KB read and the whole corpus. + """ + subtasks: dict[str, str] = {} + # Not the same test as `subtasks` being non-empty: no file means the corpus + # was never downloaded, where a file with no rubrics is a join failure the + # loader reports per problem. + found_a_parquet = False + for year in TEST_YEARS: + path = os.path.join(tests_repo, year_parquet_name(year)) + if not os.path.exists(path): + continue + found_a_parquet = True + table = pq.read_table(path, columns=["problem_id", "subtasks"]) + for problem_id, payload in zip( + table.column("problem_id").to_pylist(), + table.column("subtasks").to_pylist(), + strict=True, + ): + # NOT `payload or "{}"`, which would turn a NULL cell into a present + # but empty rubric, passing the loader's own check below and only + # surfacing per sample at grading time. + if payload is not None: + subtasks[problem_id] = payload + if not found_a_parquet: + raise FileNotFoundError( + f"No LiveOIBench test parquet found under {tests_repo!r}; " + "run `sieval dataset download liveoibench` first." + ) + return subtasks diff --git a/sieval/meta/index.json b/sieval/meta/index.json index 5a5296b5..63ea41e8 100644 --- a/sieval/meta/index.json +++ b/sieval/meta/index.json @@ -821,6 +821,30 @@ "license": "cc", "checksums": {} }, + { + "name": "liveoibench", + "display_name": "LiveOIBench", + "description": "403 olympiad problems with official tests, subtask rubrics and human rankings; 33.5 GB of tests.", + "source": [ + "hf:LiveOIBench/LiveOIBench@0e87d7c7a667979e00df41889656d73f1d3ce153", + "hf:LiveOIBench/LiveOIBench_tests@05731838fc460d3235c70506ee177aeb68a610cf", + "hf:LiveOIBench/LiveOIBench_contestants@fdcc2ba59670b06ec55ee61bc771ed855ba6dba6" + ], + "categories": [ + { + "level1": "Code", + "level2": "CodeGeneration" + } + ], + "tags": [ + "english", + "cpp", + "code-exec" + ], + "deps_group": null, + "license": "cc-by-4.0", + "checksums": {} + }, { "name": "math_500", "display_name": "MATH-500", @@ -2369,6 +2393,28 @@ "status": "stable", "reference_kind": "procedure" }, + { + "name": "liveoibench_0shot_gen", + "display_name": "LiveOIBench (0-shot)", + "description": "LiveOIBench — olympiad C++ problems, subtask-scored and ranked against human contestants.", + "dataset": "liveoibench", + "eval_mode": "gen", + "n_shot": 0, + "tags": [ + "english", + "cpp", + "code-exec" + ], + "deps_group": null, + "model_type": "chat", + "reference_impl": { + "source": "liveoibench", + "url": "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/base_judge.py", + "notes": "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), subtask scoring (base_judge.py interprete_task_result) and ranking (generate_rankings.py) are ported under sieval/community/liveoibench/. Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe -static -g, RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, and its output comparison. No checker path: the published dataset ships no checkers/ directory, so upstream's own judge compares outputs directly and the 36 min-score subtasks collapse onto the all-or-nothing rule. Upstream samples n=8 per problem and reports the best; this task defaults to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 interactive ones are filtered out by the dataset (no interactor process yet), so numbers are over 380 problems and are NOT comparable to the paper's 403-problem table. Aggregation follows upstream's overall block: relative_score is meaned within a contest and then across contests, pass_rate is the fraction of problems fully solved, and tests_passed_pct the fraction of test cases. Human ranking resolves a contest through the contestant table's own 'problems' column, which is where USACO's division split (-platinum / -combined) lives; all 22 USACO rows publish medal cutoffs but no contestant list, so they yield a medal and no percentile, and -combined yields neither -- upstream scores it from promotion thresholds the published dataset does not carry. Three IATI problems (senior-game, senior-ones, junior-twins) link against a grader whose header the prompt never shows, so they cannot compile and score 0; upstream behaves identically. Codeforces Elo is not computed. 'experimental' because no published number has been reproduced: the paper's table is over 403 problems where this scores 380, and upstream publishes no model outputs to anchor the grader against instead. Promote once an n=8 run is aligned." + }, + "status": "experimental", + "reference_kind": "procedure" + }, { "name": "math_500_0shot_gen", "display_name": "MATH-500 (0-shot, generative)", diff --git a/sieval/tasks/__init__.pyi b/sieval/tasks/__init__.pyi index 79df62e8..dc8656f4 100644 --- a/sieval/tasks/__init__.pyi +++ b/sieval/tasks/__init__.pyi @@ -130,6 +130,9 @@ from .livecodebench_code_generation_0shot_gen import ( from .livecodebench_code_generation_kshot_base_gen import ( LiveCodeBenchCodeGenerationFewShotBaseGenTask, ) +from .liveoibench_0shot_gen import ( + LiveOIBenchZeroShotGenTask, +) from .math_500_0shot_gen import ( MATH500ZeroShotGenTask, ) @@ -283,6 +286,7 @@ __all__ = [ "InverseIFEvalZeroShotGenTask", "LiveCodeBenchCodeGenerationFewShotBaseGenTask", "LiveCodeBenchCodeGenerationZeroShotGenTask", + "LiveOIBenchZeroShotGenTask", "MATH500ZeroShotGenTask", "MATHPerturbHardZeroShotGenTask", "MATHPerturbSimpleZeroShotGenTask", diff --git a/sieval/tasks/liveoibench_0shot_gen.py b/sieval/tasks/liveoibench_0shot_gen.py new file mode 100644 index 00000000..17c028dc --- /dev/null +++ b/sieval/tasks/liveoibench_0shot_gen.py @@ -0,0 +1,674 @@ +"""LiveOIBench — informatics-olympiad problems, graded in C++ with subtask credit. + +Grading runs in the evaluator service (``source="liveoibench"``), which compiles +the submission with ``g++`` and runs every official test case under the +problem's own time and memory limits. Two things make this task unlike the other +code benchmarks here: + +* **Partial credit.** A submission is not pass/fail — a subtask pays its points + only when every test in it passes, and a problem's score is the sum. The + evaluator therefore returns the whole verdict vector rather than a count. +* **A human baseline.** Every contest ships its real contestants' scores, so the + report ranks the model inside the contest it was actually sitting. + +Test data is read by the evaluator from the materialized corpus by path, not +shipped inline: a problem averages ~140 MB of test cases, which would otherwise +cross the wire once per rollout. Pass ``inline_tests=True`` when the evaluator +cannot see that volume. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import asyncio +import json +import os +import time +from collections import defaultdict +from typing import override + +import httpx +from loguru import logger + +from sieval.community.liveoibench.code_extractor import CodeExtractor +from sieval.community.liveoibench.payloads import load_code_bundle, load_subtasks +from sieval.community.liveoibench.prompts import build_prompt +from sieval.community.liveoibench.rankings import ( + build_problem_to_contest_map, + resolve_contest_id, + score_contest, +) +from sieval.community.liveoibench.scoring import interprete_task_result, total_points +from sieval.core.models import ModelOutput +from sieval.core.tasks import ( + EvalMode, + JudgementRecord, + NonRetriableSampleError, + PredictionRecord, + PromptRecord, + ReferenceImpl, + RolloutJudgement, + Task, + build_judgement_record, + build_prediction_record, + build_prompt_record, + build_rollout_judgement, + sieval_task, +) +from sieval.core.tasks.metrics import ( + DENOMINATOR_FIELD, + DENOMINATOR_REQUESTED, + SCORE_KEY_FIELD, + budget_metrics, + health_metrics, +) +from sieval.datasets import LiveOIBenchDatasetSample + +_UPSTREAM = "https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12" + +# Case count assumed when this process cannot read the test tree but the +# evaluator can. Only sizes the HTTP deadline, so it is deliberately above the +# largest published problem (429 cases, JOI-2024-JOI_spring-2024-sp_Table_Tennis; +# median 42): too generous costs a slow failure, too tight aborts a live grade. +_ASSUMED_CASES = 450 + + +def _cutoff(value) -> float | None: + """A medal cutoff cell as a float, or ``None`` when the contest publishes none. + + Upstream's ``float(x) if not pd.isna(x) else None``. A NaN surviving as a + float would read as a cutoff nothing can clear, putting the contest into the + medal denominator with the medal ``"None"`` instead of leaving it out. + """ + if value is None: + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return None if number != number else number + + +@sieval_task( + name="liveoibench_0shot_gen", + display_name="LiveOIBench (0-shot)", + description=( + "LiveOIBench — olympiad C++ problems, subtask-scored and ranked against " + "human contestants." + ), + eval_mode=EvalMode.GEN, + n_shot=0, + tags=("english", "cpp", "code-exec"), + model_type="chat", + reference_kind="procedure", + # Faithful port, no anchored reproduction yet; see `notes`. + status="experimental", + reference_impl=ReferenceImpl( + source="liveoibench", + url=f"{_UPSTREAM}/src/judges/base_judge.py", + notes=( + "Prompt (judges/problem.py get_prompt), extraction (code_extractor.py), " + "subtask scoring (base_judge.py interprete_task_result) and ranking " + "(generate_rankings.py) are ported under sieval/community/liveoibench/. " + "Execution matches BatchJudge: g++ -std=gnu++17 -Wall -O2 -pipe " + "-static -g, " + "RLIMIT_CPU/RLIMIT_AS at the problem's limits with upstream's 20% buffer, " + "and its output comparison. No checker path: the published dataset ships " + "no checkers/ directory, so upstream's own judge compares outputs directly " + "and the 36 min-score subtasks collapse onto the all-or-nothing rule. " + "Upstream samples n=8 per problem and reports the best; this task defaults " + "to n=1, so match it with n=8. Scope: the 380 'batch' problems -- the 23 " + "interactive ones are filtered out by the dataset (no interactor process " + "yet), so numbers are over 380 problems and are NOT comparable to the " + "paper's 403-problem table. Aggregation follows upstream's overall block: " + "relative_score is meaned within a contest and then across contests, " + "pass_rate is the fraction of problems fully solved, and tests_passed_pct " + "the fraction of test cases. Human ranking resolves a contest through the " + "contestant table's own 'problems' column, which is where USACO's division " + "split (-platinum / -combined) lives; all 22 USACO rows publish medal " + "cutoffs but no contestant list, so they yield a medal and no percentile, " + "and -combined yields neither -- upstream scores it from promotion " + "thresholds the published dataset does not carry. Three IATI problems " + "(senior-game, senior-ones, junior-twins) link against a grader whose " + "header the prompt never shows, so they cannot compile and score 0; " + "upstream behaves identically. Codeforces Elo is not computed. " + "'experimental' because no published number has been reproduced: the " + "paper's table is over 403 problems where this scores 380, and " + "upstream publishes no model outputs to anchor the grader against " + "instead. Promote once an n=8 run is aligned." + ), + ), +) +class LiveOIBenchZeroShotGenTask( + Task[ + LiveOIBenchDatasetSample, + PromptRecord, + ModelOutput, + PredictionRecord, + JudgementRecord, + # `float | str`: the report carries `score_key`, which names a column + # rather than measuring one. + dict[str, float | str], + ] +): + def __init__( + self, + dataset, + model, + name: str | None = None, + n: int = 1, + max_concurrency: int = 4, + inline_tests: bool = False, + ): + """*n* is the sampling budget; upstream draws 8 and reports the best of + them, which this task reproduces for whatever *n* it is given. + + *inline_tests* ships each problem's test cases in the request body + instead of naming the directory. Only for an evaluator that cannot read + the materialized corpus — it moves ~140 MB per rollout on average. + """ + super().__init__(dataset=dataset, model=model, name=name) + self._n = n + self._inline_tests = inline_tests + self._code_eval_api = os.getenv( + "SIEVAL_CODE_EVAL_API", "http://localhost:11451/evaluations" + ) + self._http_client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=max_concurrency) + ) + + @override + async def preprocess(self, raw, ctx): + prompt = build_prompt( + task=raw["task_name"], + statement=raw["problem_statement"], + time_limit=raw["time_limit"], + memory_limit=raw["memory_limit"], + starter_codes=load_code_bundle(raw["starter_code"]), + ) + # No system message: upstream's default run sends the prompt alone + # (`--system_prompt` is opt-in and supplies a per-model string). + return build_prompt_record( + [{"role": "user", "content": prompt}], + # No `reference`: the ground truth is a test suite and a subtask + # rubric, described in the judgement's extra instead. + ) + + @override + async def infer(self, pre, ctx): + return await self.model.agenerate(pre["prompt"], n=self._n) + + @override + async def postprocess(self, inf, ctx): + task_name = ctx.raw_sample["task_name"] + # `extract_code` returns ("", 'empty') for an empty fence and + # (None, 'not_found') for no fence; both normalize to None so + # `extracted` reports a miss rather than an empty submission. + return build_prediction_record( + [ + CodeExtractor.extract_code(choice, task_name)[0] or None + for choice in inf.texts + ] + ) + + @override + async def feedback(self, post, ctx): + raw = ctx.raw_sample + subtasks = load_subtasks(raw["subtasks"]) + if not subtasks: + # Without a rubric there is no score to compute, and re-inferring + # cannot produce one. + raise NonRetriableSampleError( + f"{raw['problem_id']}: no subtasks to score against" + ) + maximum = total_points(subtasks) + # Both halves touch the test directory -- a `listdir`, and in inline mode + # ~140 MB of reads. Off the event loop, and once per sample rather than + # once per rollout. + payload_tests, n_cases = await asyncio.to_thread(self._test_payload, raw) + timeout = self._request_timeout(raw, n_cases) + + if not post["rollouts"]: + # Without this the `next(...)` below raises StopIteration out of a + # coroutine, which Python re-raises naming neither sample nor cause. + raise NonRetriableSampleError( + f"{raw['problem_id']}: inference returned no rollouts to grade" + ) + + rollouts: list[RolloutJudgement] = [] + best_index, best_key = None, None + for rollout in post["rollouts"]: + idx = rollout["index"] + verdicts, case_names, msg, resources = await self._grade( + raw, rollout.get("prediction") or "", payload_tests, idx, timeout + ) + + results = [ + {"test_case": name, "correct": correct} + for name, correct in zip(case_names, verdicts, strict=True) + ] + try: + scored = interprete_task_result(results, subtasks) + except KeyError as e: + # A subtask naming a test the corpus does not contain: the two + # parquets disagree, and no rollout of this sample can be scored. + raise NonRetriableSampleError( + f"{raw['problem_id']}: subtask references unknown test case {e}" + ) from e + + relative_score = (scored["score"] / maximum * 100) if maximum else 0.0 + rollouts.append( + build_rollout_judgement( + idx, + # `correct` is the cross-task axis, so it is the olympiad's + # own all-or-nothing reading: every test passed. + scored["ace"], + score=float(scored["score"]), + metrics={ + "relative_score": relative_score, + "tests_passed_pct": scored["tests_passed"] * 100, + "ace": scored["ace"], + }, + extra={ + "msg": msg, + "subtasks": scored["subtasks"], + "n_cases": len(case_names), + "n_passed": sum(verdicts), + "resources": resources, + }, + ) + ) + # Upstream ranks candidates by score, then relative_score, then + # tests_passed_pct. Within one problem the first two are the same + # ordering, so this is (score, tests_passed). Upstream's final + # tiebreak is execution time, which is not recorded here; the lowest + # rollout index wins instead, so the pick is deterministic. + key = (scored["score"], scored["tests_passed"]) + if best_key is None or key > best_key: + best_index, best_key = idx, key + + best = next(r for r in rollouts if r["index"] == best_index) + return True, build_judgement_record( + None, # the reference is a test suite plus a rubric, not a value + rollouts, + score=best["score"], + metrics={ + "relative_score": best["metrics"]["relative_score"], + "tests_passed_pct": best["metrics"]["tests_passed_pct"], + "ace": best["metrics"]["ace"], + }, + extra={ + "problem_id": raw["problem_id"], + "contest_id": raw["contest_id"], + "task_name": raw["task_name"], + "total_points": maximum, + "n_subtasks": len(subtasks), + "best_rollout": best_index, + "task_type": raw["task_type"], + }, + ) + + def _test_payload(self, raw) -> tuple[dict, int]: + """The request's test half, and how many cases it names. + + A directory for the evaluator to read, or the cases inline. The count + only sizes the HTTP deadline — the evaluator decides the real case list. + Blocking; called from a worker thread. + """ + tests_dir = raw["tests_dir"] + try: + stems = sorted( + name[: -len(".in")] + for name in os.listdir(tests_dir) + if name.endswith(".in") + ) + except OSError: + if self._inline_tests: + raise + # Directory mode only needs the evaluator to see the tree, not this + # process; fall back to a deadline wide enough for a large problem. + return {"test_dir": tests_dir}, _ASSUMED_CASES + + if not self._inline_tests: + return {"test_dir": tests_dir}, len(stems) + + inputs, outputs, names = [], [], [] + for stem in stems: + output_path = os.path.join(tests_dir, f"{stem}.out") + if not os.path.exists(output_path): + raise NonRetriableSampleError( + f"{raw['problem_id']}: test {stem!r} has no .out in {tests_dir}" + ) + input_path = os.path.join(tests_dir, f"{stem}.in") + with open(input_path, encoding="utf-8", errors="replace") as f: + inputs.append(f.read()) + with open(output_path, encoding="utf-8", errors="replace") as f: + outputs.append(f.read()) + names.append(stem) + payload = {"inputs": inputs, "outputs": outputs, "names": names} + return {"test": payload}, len(names) + + async def _grade( + self, + raw, + code: str, + payload_tests: dict, + idx: int, + timeout: httpx.Timeout, + ) -> tuple[list[bool], list[str], str, dict]: + """One rollout through the evaluator; returns verdicts and their names.""" + task_name = raw["task_name"] + # Compiled with the submission when the problem ships one; the header has + # to be on disk for the submission's own `#include` to resolve. + files = load_code_bundle(raw["grader_code"]) + + try: + resp = await self._http_client.post( + self._code_eval_api, + json={ + "uuid": f"{raw['problem_id']}-{idx}-{time.perf_counter_ns()}", + "source": "liveoibench", + "lang": "cpp", + # An unextractable answer is "" on the wire, so the evaluator + # still runs it and reports a compile error -- a real verdict + # rather than a skipped rollout. + "code": code, + "entry_filename": f"{task_name}.cpp", + "files": files, + # The problem's own limit, per case; the evaluator applies + # upstream's 20% buffer on top. + "timeout_per_case": float(raw["time_limit"]), + "memory_limit": int(raw["memory_limit"]), + **payload_tests, + }, + timeout=timeout, + ) + resp.raise_for_status() + res = resp.json() + msg = res["msg"] + data = res["data"] or {} + except Exception as e: + logger.warning( + "Evaluation error for {} rollout {}: [{}] {}", + raw["problem_id"], + idx, + type(e).__name__, + e, + ) + raise + + verdicts = data.get("case_verdicts") + case_names = data.get("case_names") + if verdicts is None or case_names is None: + # Reached an evaluator without the C++ path (or an older one): there + # is no verdict vector to score subtasks from, and `status` alone + # would silently score every partial solution as zero. + raise RuntimeError( + f"evaluator returned no per-case verdicts for {raw['problem_id']!r} " + f"(msg: {msg}); it needs the liveoibench C++ source " + "(vendor/code-evaluator/VENDORED.md)." + ) + if not case_names: + # An empty verdict vector scores every subtask at zero, so say what + # it is instead: the evaluator saw the directory and found nothing. + raise NonRetriableSampleError( + f"{raw['problem_id']}: the evaluator found no test cases in " + f"{raw['tests_dir']!r} (msg: {msg}). Re-run " + "`python scripts/materialize_liveoibench_tests.py --overwrite`." + ) + + resources = { + key: value + for key, value in data.items() + if key not in ("n_cases", "n_passed", "case_verdicts", "case_names") + } + return verdicts, case_names, msg, resources + + def _request_timeout(self, raw, n_cases: int) -> httpx.Timeout: + """HTTP deadline, outside every budget the evaluator can spend. + + Compilation is bounded at the evaluator's 60s default and each case at + its own 120s wall, which only a process burning no CPU can reach. The + suite runs at most 4 cases at a time there, so the bound is derived from + the case count rather than assumed. + + ``pool=None`` because a bare float would also cap the wait for a free + connection, and these deadlines differ by an order of magnitude across + problems: a ten-case problem queued behind hour-long ones would fail on + the pool without ever being graded. + """ + per_case = min(120.0, float(raw["time_limit"]) * 1.2 + 5.0) + return httpx.Timeout(60.0 + per_case * n_cases + 30.0, pool=None) + + @override + async def report(self, finals, fails): + total = len(finals) + len(fails) + contestants = self._load_contestants() + by_contest = self._group_by_contest(finals, fails, contestants) + + # Upstream's `overall` block (generate_rankings.py): `relative_score` is + # meaned inside a contest and then across contests, so a 10-problem round + # does not outweigh a 1-problem one. Measured over the 380 scored + # problems: 72 contests sized 1..10, the largest being + # JOI-2023-JOI_spring at 10, then eleven USACO rounds at 9 each. So the + # two weightings are not interchangeable. + contest_relative = [ + sum(r["relative_score"] for r in rows) / len(rows) + for rows in by_contest.values() + ] + contest_tests = [ + sum(r["tests_passed_pct"] for r in rows) / len(rows) + for rows in by_contest.values() + ] + relative = ( + sum(contest_relative) / len(contest_relative) if contest_relative else 0.0 + ) + # Upstream's `pass_rate` is problem-level -- `total_solved / + # total_problem_count` -- and counts a problem only when every test in it + # passed. A failed sample counts as unsolved rather than dropping out of + # the denominator, which is what DENOMINATOR_REQUESTED means here. + solved = sum(1 for rows in by_contest.values() for r in rows if r["solved"]) + + metrics: dict[str, float | str] = { + "score": relative, + "fails": float(len(fails)), + "relative_score": relative, + "pass_rate": (solved / total * 100) if total else 0.0, + "tests_passed_pct": ( + sum(contest_tests) / len(contest_tests) if contest_tests else 0.0 + ), + "n_problems": float(total), + "n_scored": float(len(finals)), + "n_contests": float(len(by_contest)), + SCORE_KEY_FIELD: "relative_score", + DENOMINATOR_FIELD: DENOMINATOR_REQUESTED, + } + metrics |= self._human_metrics(by_contest, contestants) + metrics |= budget_metrics( + [f.feedback_result["n_rollouts"] for f in finals], + n=self._n, + # `k = n`: the headline is best-of-n, not pass@k, so every rollout + # drawn is a candidate for the one that scores. + k=self._n, + ) + return metrics | health_metrics(finals) + + def _group_by_contest(self, finals, fails, contestants) -> dict[str, list[dict]]: + """Every requested problem, keyed by the contest it is *ranked* in. + + Not the contest its id names: USACO splits one round into a ``-platinum`` + and a ``-combined`` ranking, and only the contestant table says which + problems went where. :func:`resolve_contest_id` walks upstream's ladder, + falling back to the id-derived contest when the table has nothing. + + Failed samples ride along at zero — the denominator is every sample + requested, so a sample that never produced a judgement has to depress its + contest's mean rather than vanish from it. + """ + problem_to_contest = build_problem_to_contest_map(contestants.values()) + available = set(contestants) + + grouped: dict[str, list[dict]] = defaultdict(list) + unplaced = 0 + + def place(ctx, result) -> None: + nonlocal unplaced + raw = ctx.raw_sample + if raw is None: + unplaced += 1 + return + contest_id = ( + resolve_contest_id( + raw["problem_id"], raw["contest_id"], problem_to_contest, available + ) + or raw["contest_id"] + ) + grouped[contest_id].append( + { + "task_name": raw["task_name"], + "score": float(result["score"]) if result else 0.0, + "relative_score": ( + result["metrics"]["relative_score"] if result else 0.0 + ), + "tests_passed_pct": ( + result["metrics"]["tests_passed_pct"] if result else 0.0 + ), + "solved": bool(result["metrics"]["ace"]) if result else False, + # The score means take a failure at zero; the human + # comparison must not -- see `_human_metrics`. + "scored": result is not None, + } + ) + + for ctx in finals: + place(ctx, ctx.feedback_result) + for ctx in fails: + # A failed context never contributes a judgement, even when it + # produced one before failing: the run did not accept it. + place(ctx, None) + + if unplaced: + logger.warning( + "{} sample(s) carried no dataset row and are absent from the " + "per-contest means; they still count in the denominator.", + unplaced, + ) + return dict(grouped) + + def _human_metrics(self, by_contest, contestants) -> dict[str, float]: + """Percentile and medals, per contest, against that contest's field. + + Upstream re-totals the humans over exactly the tasks the model was scored + on, so a contest whose interactive problems were filtered out still + compares like with like. Contests are averaged unweighted, as upstream + averages its per-contest percentiles. + + "Scored on" is why a **failed** sample is dropped here while it still + counts at zero in ``relative_score`` and ``pass_rate``: feeding it in + would penalise twice, once on the model's total and once by adding the + human column for a problem that was never graded. A contest whose + problems all failed therefore reports no percentile at all. + + The two counts are always reported once a contestant table was read, so + ``n_contests_ranked`` of 0 reads as measured rather than as missing — + every USACO contest in the release publishes cutoffs but no contestant + list, so a USACO-only run legitimately ranks nothing and still medals. + """ + if not contestants: + return {} + + percentiles: list[float] = [] + medals: list[str] = [] + unmatched: list[str] = [] + for contest_id, rows in by_contest.items(): + contest = contestants.get(contest_id) + if contest is None: + unmatched.append(contest_id) + continue + scored_rows = [row for row in rows if row["scored"]] + if not scored_rows: + continue + model_scores = {row["task_name"]: row["score"] for row in scored_rows} + if len(model_scores) != len(scored_rows): + # Two problems sharing a name inside one contest collapse into a + # single entry, dropping the other from both the model total and + # the matched columns. No published contest does this (0 of 72), + # so say so rather than score a quietly smaller contest. + logger.warning( + "{}: {} scored problems share only {} distinct task name(s); " + "the human comparison for this contest is incomplete.", + contest_id, + len(scored_rows), + len(model_scores), + ) + result = score_contest( + contest["rankings"], + model_scores, + contest_id=contest_id, + gold_cutoff=contest["gold_cutoff"], + silver_cutoff=contest["silver_cutoff"], + bronze_cutoff=contest["bronze_cutoff"], + ) + if result["human_percentile"] is not None: + percentiles.append(result["human_percentile"]) + if result["medal"] is not None: + medals.append(result["medal"]) + + if unmatched: + logger.warning( + "{} contest(s) had no contestant record and are excluded from the " + "human metrics: {}", + len(unmatched), + ", ".join(sorted(unmatched)), + ) + + metrics: dict[str, float] = { + "n_contests_ranked": float(len(percentiles)), + "n_contests_medalled": float(len(medals)), + } + if percentiles: + metrics["human_percentile"] = sum(percentiles) / len(percentiles) + if medals: + metrics["medal_rate"] = ( + sum(1 for m in medals if m in ("Gold", "Silver", "Bronze")) + / len(medals) + * 100 + ) + metrics["gold_rate"] = ( + sum(1 for m in medals if m == "Gold") / len(medals) * 100 + ) + return metrics + + def _load_contestants(self) -> dict[str, dict]: + """The contestant table, keyed by contest, or ``{}`` when unavailable.""" + path = getattr(self.dataset, "contestants_path", None) + if not path or not os.path.exists(path): + logger.warning( + "No contestant results at {}; reporting scores without human ranking.", + path, + ) + return {} + + import pyarrow.parquet as pq + + table = pq.read_table(path) + contestants: dict[str, dict] = {} + for row in table.to_pylist(): + try: + rankings = json.loads(row["contestants_ranking"] or "[]") + except json.JSONDecodeError: + continue + contestants[row["contest_id"]] = { + # `contest_id` and `problems` are what builds the problem-to-contest + # map; upstream reads the division split out of the same column. + "contest_id": row["contest_id"], + "problems": row.get("problems"), + "rankings": rankings, + "gold_cutoff": _cutoff(row["gold_cutoff"]), + "silver_cutoff": _cutoff(row["silver_cutoff"]), + "bronze_cutoff": _cutoff(row["bronze_cutoff"]), + } + return contestants + + @override + async def shutdown(self): + await self._http_client.aclose() diff --git a/tests/unit/community/liveoibench/__init__.py b/tests/unit/community/liveoibench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/community/liveoibench/test_prompts.py b/tests/unit/community/liveoibench/test_prompts.py new file mode 100644 index 00000000..7b32fde4 --- /dev/null +++ b/tests/unit/community/liveoibench/test_prompts.py @@ -0,0 +1,80 @@ +"""Unit tests for LiveOIBench prompt construction. + +The prompt is the benchmark's input, so these pin the exact strings upstream +assembles -- including the two things a tidy-up would change first: the limits +rendering as the floats the parquet carries, and the grader block appearing only +when all three attachment files are present. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +from sieval.community.liveoibench.prompts import build_prompt + +STATEMENT = "# Beech Tree\n\nGiven $N$ nodes...\n" + + +def test_the_fence_is_named_after_the_task(): + prompt = build_prompt("beechtree", STATEMENT, 1.5, 2048.0) + assert prompt.startswith( + "Given a competition problem below, write a solution in C++ that solves " + "all the subtasks. Make sure to wrap your code in '```beechtree.cpp' and " + "'```' Markdown delimiters.\n\n" + ) + + +def test_the_statement_is_delimited_verbatim(): + prompt = build_prompt("t", STATEMENT, 1.0, 256.0) + assert f"[BEGIN PROBLEM]\n{STATEMENT}[END PROBLEM]\n" in prompt + + +def test_limits_render_as_the_floats_the_dataset_carries(): + # Upstream interpolates the parquet value straight in, so 2048.0 -- not 2048 + # -- is what every published run showed the model. + prompt = build_prompt("t", STATEMENT, 1.5, 2048.0) + assert "Time limit: 1.5 seconds\nMemory limit: 2048.0 MB\n" in prompt + + +def test_absent_limits_render_as_unknown(): + prompt = build_prompt("t", STATEMENT, None, None) + assert "Time limit: unknown seconds\nMemory limit: unknown MB\n" in prompt + + +def test_without_attachments_the_plain_instruction_is_appended(): + prompt = build_prompt("t", STATEMENT, 1.0, 256.0) + assert prompt.endswith( + "Generate a solution in C++ that solves the task. Make sure to wrap your " + "code in '```t.cpp' and '```' Markdown delimiters.\n\n" + ) + assert "grader.cpp" not in prompt + + +def test_the_grader_block_needs_all_three_files(): + partial = {"grader.cpp": "int main(){}\n", "t.h": "void f();\n"} # no t.cpp + assert "```grader.cpp" not in build_prompt("t", STATEMENT, 1.0, 256.0, partial) + + whole = partial | {"t.cpp": "void f(){}\n"} + prompt = build_prompt("t", STATEMENT, 1.0, 256.0, whole) + assert ( + "We are going to grade your solution using the following grader.cpp" in prompt + ) + assert "```grader.cpp\nint main(){}\n```\n\n" in prompt + assert "```t.h\nvoid f();\n```\n\n" in prompt + assert "```t.cpp\nvoid f(){}\n```\n\n" in prompt + + +def test_stub_cpp_stands_in_for_grader_cpp(): + # Upstream falls back to `stub.cpp` when a problem ships no `grader.cpp`, and + # still labels the block `grader.cpp`. + bundle = { + "stub.cpp": "int main(){}\n", + "t.h": "void f();\n", + "t.cpp": "void f(){}\n", + } + prompt = build_prompt("t", STATEMENT, 1.0, 256.0, bundle) + assert "```grader.cpp\nint main(){}\n```\n\n" in prompt + + +def test_a_task_name_with_a_dash_survives_into_the_fence(): + # `CEOI-2023-contest-grading-server` -> task `grading-server`. + prompt = build_prompt("grading-server", STATEMENT, 1.0, 256.0) + assert "'```grading-server.cpp'" in prompt diff --git a/tests/unit/community/liveoibench/test_rankings.py b/tests/unit/community/liveoibench/test_rankings.py new file mode 100644 index 00000000..5487e36c --- /dev/null +++ b/tests/unit/community/liveoibench/test_rankings.py @@ -0,0 +1,309 @@ +"""Unit tests for ranking a model against a contest's human contestants. + +The load-bearing behaviour is that humans are re-totaled over **only** the tasks +the model was scored on: sieval filters interactive problems out, so a contest is +often partial, and comparing a partial model total against full human totals +would understate every model by however much it skipped. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import json + +from sieval.community.liveoibench.rankings import ( + build_problem_to_contest_map, + calculate_percentile, + identify_task_columns, + medal_from_cutoffs, + normalize_contest_identifier, + normalize_name, + resolve_contest_id, + score_contest, +) + +# Shape of the published `contestants_ranking` payload. +RANKINGS = [ + { + "Rank": 1, + "Contestant": "A", + "Country": "CN", + "alpha": 100, + "beta": 100, + "Total": 200, + }, + { + "Rank": 2, + "Contestant": "B", + "Country": "US", + "alpha": 100, + "beta": 20, + "Total": 120, + }, + { + "Rank": 3, + "Contestant": "C", + "Country": "PL", + "alpha": 40, + "beta": 0, + "Total": 40, + }, + {"Rank": 4, "Contestant": "D", "Country": "JP", "alpha": 0, "beta": 0, "Total": 0}, +] + + +def test_percentile_counts_only_contestants_strictly_below(): + # Upstream is `model_score > human_scores`, so tying the field beats nobody. + assert calculate_percentile(40, [0, 40, 120, 200]) == 25.0 + assert calculate_percentile(0, [0, 40, 120, 200]) == 0.0 + assert calculate_percentile(201, [0, 40, 120, 200]) == 100.0 + + +def test_percentile_of_an_empty_field_is_unknown_not_zero(): + assert calculate_percentile(100, []) is None + + +def test_medal_ladder_is_inclusive_at_each_cutoff(): + assert medal_from_cutoffs(236, 236, 191, 145) == "Gold" + assert medal_from_cutoffs(235, 236, 191, 145) == "Silver" + assert medal_from_cutoffs(145, 236, 191, 145) == "Bronze" + assert medal_from_cutoffs(144, 236, 191, 145) == "None" + + +def test_a_contest_publishing_no_cutoffs_yields_no_medal_rather_than_none_the_string(): + assert medal_from_cutoffs(500, None, None, None) is None + + +def test_header_columns_are_not_mistaken_for_tasks(): + mapping = identify_task_columns(list(RANKINGS[0])) + assert set(mapping) == {"alpha", "beta"} + + +def test_boi_scores_by_day_so_day_columns_are_tasks_there(): + columns = ["Rank", "Contestant", "Day1", "Day2"] + assert identify_task_columns(columns, "BOI-2025-contest") == { + "day1": "Day1", + "day2": "Day2", + } + assert identify_task_columns(columns, "IOI-2025-contest") == {} + + +def test_normalize_name_strips_everything_but_alphanumerics(): + assert normalize_name("Cow-libi (Silver)") == "cowlibisilver" + + +def test_humans_are_retotaled_over_only_the_tasks_the_model_attempted(): + # Model scored 100 on `alpha` alone. Against alpha-only human totals + # (100, 100, 40, 0) it beats two; against full totals it would beat one. + result = score_contest(RANKINGS, {"alpha": 100}) + assert result["model_total"] == 100 + assert result["matched_columns"] == ["alpha"] + assert result["human_percentile"] == 50.0 + + +def test_scoring_every_task_uses_every_column(): + result = score_contest(RANKINGS, {"alpha": 100, "beta": 100}) + assert result["model_total"] == 200 + assert sorted(result["matched_columns"]) == ["alpha", "beta"] + # Ties with the top contestant, so it beats the other three. + assert result["human_percentile"] == 75.0 + + +def test_task_names_match_columns_case_and_punctuation_insensitively(): + rankings = [{"Rank": 1, "Cow-libi": 100, "Total": 100}] + result = score_contest(rankings, {"cow_libi": 50}) + assert result["matched_columns"] == ["Cow-libi"] + + +def test_unmatched_task_names_fall_back_to_the_contest_total(): + result = score_contest(RANKINGS, {"gamma": 130}) + assert result["matched_columns"] == ["Total"] + assert result["model_total"] == 130 + # Against Total (200, 120, 40, 0) a 130 beats three. + assert result["human_percentile"] == 75.0 + + +def test_a_field_with_no_usable_column_at_all_reports_no_percentile(): + result = score_contest([{"Rank": 1}], {"gamma": 10}) + assert result["human_percentile"] is None + assert result["model_total"] == 10 + + +def test_non_numeric_and_missing_cells_count_as_zero(): + rankings = [ + {"alpha": "-"}, # withdrew + {"alpha": None}, + {}, # column absent for this row + {"alpha": "50"}, # numeric strings still count + ] + result = score_contest(rankings, {"alpha": 10}) + assert result["human_percentile"] == 75.0 + + +def test_medals_use_the_contests_published_cutoffs(): + result = score_contest( + RANKINGS, + {"alpha": 100, "beta": 100}, + gold_cutoff=200.0, + silver_cutoff=120.0, + bronze_cutoff=40.0, + ) + assert result["medal"] == "Gold" + assert result["n_contestants"] == 4 + + +# --------------------------------------------------------------------------- # +# Contest identity — USACO splits one round into two rankings +# --------------------------------------------------------------------------- # +def test_the_contest_map_reads_the_division_split_out_of_the_problems_column(): + """The only place the split is stated is the contestant row's own list.""" + rows = [ + { + "contest_id": "USACO-2025-January_Contest-platinum", + "problems": json.dumps(["USACO-2025-January_Contest-platinum_Cow"]), + }, + { + "contest_id": "USACO-2025-January_Contest-combined", + "problems": json.dumps( + [ + "USACO-2025-January_Contest-bronze_Moo", + "USACO-2025-January_Contest-gold_Hay", + ] + ), + }, + {"contest_id": "IOI-2025-contest", "problems": ["IOI-2025-contest-beechtree"]}, + ] + mapping = build_problem_to_contest_map(rows) + assert ( + mapping["USACO-2025-January_Contest-platinum_Cow"] + == "USACO-2025-January_Contest-platinum" + ) + assert ( + mapping["USACO-2025-January_Contest-bronze_Moo"] + == "USACO-2025-January_Contest-combined" + ) + assert mapping["IOI-2025-contest-beechtree"] == "IOI-2025-contest" + + +def test_a_division_listed_by_the_wrong_row_is_not_mapped(): + """Upstream's filter: a `-combined` row never claims a platinum problem.""" + rows = [ + { + "contest_id": "USACO-2025-January_Contest-combined", + "problems": json.dumps(["USACO-2025-January_Contest-platinum_Cow"]), + } + ] + assert build_problem_to_contest_map(rows) == {} + + +def test_the_contestant_table_wins_over_the_id_derived_contest(): + """Upstream reads the mapping out of the table first and only derives one + when the table lists nothing. + + On the published data the two routes happen to agree everywhere, so nothing + else here would notice the table being ignored — but the table is the only + one of the two that can follow a release that regroups a round. + """ + mapping = build_problem_to_contest_map( + [{"contest_id": "IOI-2025-day2", "problems": ["IOI-2025-contest-beechtree"]}] + ) + assert ( + resolve_contest_id( + "IOI-2025-contest-beechtree", + "IOI-2025-contest", + mapping, + {"IOI-2025-day2", "IOI-2025-contest"}, + ) + == "IOI-2025-day2" + ) + + +def test_an_unlisted_usaco_problem_falls_back_to_its_division_suffix(): + resolved = resolve_contest_id( + "USACO-2025-January_Contest-platinum_Cow", + "USACO-2025-January_Contest", + {}, + {"USACO-2025-January_Contest-platinum"}, + ) + assert resolved == "USACO-2025-January_Contest-platinum" + + +def test_a_non_usaco_problem_resolves_to_the_contest_its_id_names(): + assert ( + resolve_contest_id("IOI-2025-contest-beechtree", "IOI-2025-contest", {}, set()) + == "IOI-2025-contest" + ) + + +def test_the_cco_rename_falls_back_to_the_identifier_the_table_actually_uses(): + """Upstream renames the CCO rounds and the published table does not, so the + ladder has to come back to the raw identifier or CCO stops matching.""" + raw = "CCO-2024-Canadian_Computing_Competition_Senior" + assert normalize_contest_identifier(raw) == "CCO-2024-Senior" + assert resolve_contest_id(f"{raw}-x", raw, {}, {raw}) == raw + + +# --------------------------------------------------------------------------- # +# Recalculated_Total — upstream prefers it wherever it exists +# --------------------------------------------------------------------------- # +def test_a_recalculated_total_column_replaces_the_per_task_sum(): + """The two are not the same number: the column is the contest's own + re-derivation, and on the published data it differs from the per-task sum + for 20-83% of contestants on each of the 10 contests that carry it.""" + rankings = [ + {"alpha": 0, "beta": 0, "Recalculated_Total": 300}, + {"alpha": 100, "beta": 100, "Recalculated_Total": 10}, + ] + result = score_contest(rankings, {"alpha": 100, "beta": 0}) + assert result["matched_columns"] == ["Recalculated_Total"] + # Against the column the model's 100 beats only the 10 -> 50th. Summing the + # task columns would have compared it against 0 and 200 instead; the column + # name above is what discriminates, the percentile pins the values. + assert result["human_percentile"] == 50.0 + + +def test_the_recalculated_branch_drops_unparseable_rows_rather_than_zeroing_them(): + """Upstream's `.dropna()`. Zeroing instead would invent contestants the + model outscores, which inflates the percentile.""" + rankings = [ + {"Recalculated_Total": 300}, + {"Recalculated_Total": None}, + {"Recalculated_Total": "n/a"}, + ] + result = score_contest(rankings, {"alpha": 10}) + # One usable contestant, who beat the model. Zeroing the other two would + # have reported 66.7 instead. + assert result["human_percentile"] == 0.0 + + +def test_rows_without_the_column_still_take_the_per_task_branch(): + result = score_contest(RANKINGS, {"alpha": 100, "beta": 100}) + assert result["matched_columns"] == ["alpha", "beta"] + + +# --------------------------------------------------------------------------- # +# Contests that publish cutoffs but no contestants +# --------------------------------------------------------------------------- # +def test_an_empty_field_still_awards_the_medal_its_cutoffs_imply(): + """Upstream's `df.empty` branch. Every USACO row in the release is this + shape, so reporting no medal here would lose all 22 of them.""" + result = score_contest( + [], {"alpha": 100}, gold_cutoff=90.0, silver_cutoff=60.0, bronze_cutoff=30.0 + ) + assert result["human_percentile"] is None + assert result["medal"] == "Gold" + assert result["n_contestants"] == 0 + + +def test_a_usaco_combined_contest_reports_nothing(): + """Its metric is a promotion threshold that the published dataset does not + carry; upstream returns no medal and no percentile for it.""" + result = score_contest( + [], + {"bronze_Moo": 100}, + contest_id="USACO-2025-January_Contest-combined", + gold_cutoff=800.0, + silver_cutoff=750.0, + bronze_cutoff=700.0, + ) + assert result["human_percentile"] is None + assert result["medal"] is None diff --git a/tests/unit/community/liveoibench/test_scoring.py b/tests/unit/community/liveoibench/test_scoring.py new file mode 100644 index 00000000..77f4549f --- /dev/null +++ b/tests/unit/community/liveoibench/test_scoring.py @@ -0,0 +1,108 @@ +"""Unit tests for LiveOIBench subtask scoring. + +The arithmetic is the benchmark: a subtask is all-or-nothing, so a submission +that passes most tests of every subtask can score zero while another that passes +fewer tests scores points. Pinning that here keeps a "reasonable" partial-credit +rewrite from quietly becoming a different benchmark. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import pytest + +from sieval.community.liveoibench.scoring import interprete_task_result, total_points + +SUBTASKS = { + "1": {"task": "Subtask 1", "score": 0, "testcases": ["s0"]}, + "2": {"task": "Subtask 2", "score": 30, "testcases": ["a1", "a2"]}, + "3": {"task": "Subtask 3", "score": 70, "testcases": ["b1", "b2"]}, +} + + +def _results(**passed: bool) -> list[dict]: + return [{"test_case": name, "correct": ok} for name, ok in passed.items()] + + +def test_a_subtask_pays_only_when_every_test_in_it_passes(): + scored = interprete_task_result( + _results(s0=True, a1=True, a2=True, b1=True, b2=False), SUBTASKS + ) + # Subtask 3 lost one test of two, so its 70 points are gone entirely. + assert scored["score"] == 30 + assert scored["subtasks"]["3"]["score"] == 0 + assert scored["subtasks"]["3"]["passed"] == 0.5 + + +def test_more_tests_passed_can_still_score_less(): + # 4 of 5 tests, spread so that both paying subtasks are broken -> 0 points. + spread = interprete_task_result( + _results(s0=True, a1=True, a2=False, b1=True, b2=True), SUBTASKS + ) + # 3 of 5 tests, but one whole subtask intact -> 30 points. + focused = interprete_task_result( + _results(s0=False, a1=True, a2=True, b1=True, b2=False), SUBTASKS + ) + assert spread["tests_passed"] > focused["tests_passed"] + assert spread["score"] == 70 # subtask 3 whole + assert focused["score"] == 30 + + +def test_ace_and_tests_passed_span_every_result(): + scored = interprete_task_result( + _results(s0=True, a1=True, a2=True, b1=True, b2=True), SUBTASKS + ) + assert scored["score"] == 100 + assert scored["tests_passed"] == 1 + assert scored["ace"] is True + + +def test_a_zero_point_sample_subtask_does_not_change_the_score(): + without = interprete_task_result( + _results(s0=False, a1=True, a2=True, b1=True, b2=True), SUBTASKS + ) + assert without["score"] == 100 + assert without["ace"] is False, "a failed sample test still breaks the ace flag" + + +def test_total_is_rounded_not_truncated(): + # min-score grading with a checker score of 0.5 on a 45-point subtask -> 22.5. + subtasks = {"1": {"score": 45, "grading": "min-score", "testcases": ["a", "b"]}} + results = [ + {"test_case": "a", "correct": False, "score": 0.5}, + {"test_case": "b", "correct": True, "score": 1.0}, + ] + assert interprete_task_result(results, subtasks)["score"] == 22 + + +def test_min_score_collapses_to_all_or_nothing_without_a_checker(): + # The published dataset ships no checkers, so no result carries a `score` and + # upstream's `.get("score", 0)` reads 0 -- the same verdict as the default + # rule. Pinned because a port that invented per-test fractions here would + # silently pay partial credit upstream does not. + subtasks = {"1": {"score": 45, "grading": "min-score", "testcases": ["a", "b"]}} + partial = interprete_task_result( + [{"test_case": "a", "correct": True}, {"test_case": "b", "correct": False}], + subtasks, + ) + assert partial["score"] == 0 + whole = interprete_task_result( + [{"test_case": "a", "correct": True}, {"test_case": "b", "correct": True}], + subtasks, + ) + assert whole["score"] == 45 + + +def test_an_empty_result_set_scores_zero_rather_than_dividing_by_zero(): + scored = interprete_task_result([], {}) + assert scored == {"subtasks": {}, "score": 0, "tests_passed": 0, "ace": False} + + +def test_a_subtask_naming_an_unknown_test_raises(): + # The two parquets are out of step; scoring it would silently understate. + with pytest.raises(KeyError): + interprete_task_result([{"test_case": "a", "correct": True}], SUBTASKS) + + +def test_total_points_sums_the_rubric(): + assert total_points(SUBTASKS) == 100 + assert total_points({}) == 0 diff --git a/tests/unit/datasets/test_liveoibench.py b/tests/unit/datasets/test_liveoibench.py new file mode 100644 index 00000000..50a08f23 --- /dev/null +++ b/tests/unit/datasets/test_liveoibench.py @@ -0,0 +1,239 @@ +"""Unit tests for the LiveOIBench dataset loader. + +Covers the three-repo join (problems + subtasks + contestants), the derived +columns the task reads, and the two guards that turn a silent under-run into an +error: a missing test corpus and an empty filter. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import json +import os + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from sieval.datasets.liveoibench import ( + LiveOIBenchDataset, + parse_problem_id, + problem_tests_dir, + year_parquet_name, +) + +BATCH_ID = "IOI-2025-contest-beechtree" +INTERACTIVE_ID = "IOI-2025-contest-longesttrip" +DASHED_ID = "CEOI-2024-contest-grading-server" + +SUBTASKS = {"1": {"task": "Subtask 1", "score": 100, "testcases": ["t1"]}} + + +def _problem(problem_id: str, task_type: str = "batch") -> dict: + # The year is read out of `problem_id`, the way the loader reads it. + _, _, _, task = parse_problem_id(problem_id) + return { + "id": 0, + "problem_id": problem_id, + "competition": problem_id.split("-")[0], + "contest": "contest", + "task_name": task, + "problem_statement": f"# {task}\n", + "time_limit": 1.5, + "memory_limit": 2048.0, + "task_type": task_type, + "difficulty": 12, + "algorithms": '["greedy"]', + "setup_script": "", + "evaluation_script": "", + "grader_code": "", + "starter_code": "", + } + + +def _stage(root, problems: list[dict], *, with_tests: bool = True) -> str: + """Write the three staged repos the way `sieval dataset download` would.""" + problems_dir = root / "LiveOIBench" / "data" + problems_dir.mkdir(parents=True, exist_ok=True) + pq.write_table( + pa.Table.from_pylist(problems), problems_dir / "liveoibench_v1.parquet" + ) + + tests_repo = root / "LiveOIBench_tests" + tests_repo.mkdir(parents=True, exist_ok=True) + by_year: dict[str, list[dict]] = {} + for problem in problems: + year = parse_problem_id(problem["problem_id"])[1] + by_year.setdefault(year, []).append( + { + "id": 0, + "problem_id": problem["problem_id"], + "subtasks": json.dumps(SUBTASKS), + } + ) + for year, rows in by_year.items(): + pq.write_table(pa.Table.from_pylist(rows), tests_repo / year_parquet_name(year)) + + contestants_dir = root / "LiveOIBench_contestants" / "data" + contestants_dir.mkdir(parents=True, exist_ok=True) + pq.write_table( + pa.Table.from_pylist( + [ + { + "contest_id": "IOI-2025-contest", + "gold_cutoff": 90.0, + "silver_cutoff": 60.0, + "bronze_cutoff": 30.0, + "division": "Division 1", + "contestants_ranking": json.dumps([{"Rank": 1, "beechtree": 100}]), + "date": "2025-08-01", + "num_contestants": 1, + "problems": json.dumps([BATCH_ID]), + } + ] + ), + contestants_dir / "contest_results.parquet", + ) + + tests_root = tests_repo / "materialized" + if with_tests: + for problem in problems: + case_dir = problem_tests_dir(str(tests_root), problem["problem_id"]) + os.makedirs(case_dir, exist_ok=True) + with open(os.path.join(case_dir, "t1.in"), "w") as f: + f.write("1 2\n") + with open(os.path.join(case_dir, "t1.out"), "w") as f: + f.write("3\n") + return str(root / "LiveOIBench") + + +# --------------------------------------------------------------------------- # +# Identifier parsing +# --------------------------------------------------------------------------- # +def test_the_task_token_keeps_its_dashes(): + # `task_name` is everything after competition/year/round, so a hyphenated + # task name must not be truncated -- it names the prompt's code fence. + assert parse_problem_id(DASHED_ID) == ("CEOI", "2024", "contest", "grading-server") + + +def test_a_malformed_identifier_raises(): + with pytest.raises(ValueError, match="expected at least 4 tokens"): + parse_problem_id("IOI-2025-contest") + + +def test_tests_live_in_upstreams_reconstructed_layout(): + # Matching upstream means an existing `process_dataset.py` tree can be reused. + assert problem_tests_dir("/root", BATCH_ID) == ( + "/root/IOI/2025/contest/beechtree/tests" + ) + + +# --------------------------------------------------------------------------- # +# Loading and joining +# --------------------------------------------------------------------------- # +def test_load_joins_subtasks_and_derives_the_columns_the_task_reads(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + dataset = LiveOIBenchDataset(name_or_path=path) + + assert dataset.test_set is not None + (row,) = list(dataset.test_set) + assert json.loads(row["subtasks"]) == SUBTASKS + assert row["contest_id"] == "IOI-2025-contest" + assert row["tests_dir"].endswith("IOI/2025/contest/beechtree/tests") + assert row["task_name"] == "beechtree" + + +def test_the_contestant_parquet_is_reachable_from_the_dataset(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + dataset = LiveOIBenchDataset(name_or_path=path) + assert dataset.contestants_path is not None + assert os.path.exists(dataset.contestants_path) + + +def test_interactive_problems_are_filtered_out_by_default(tmp_path): + # Filtered rather than scored zero: the evaluator has no interactor process, + # and a zero would understate every model without saying so. + path = _stage( + tmp_path, + [_problem(BATCH_ID), _problem(INTERACTIVE_ID, task_type="interactive")], + ) + default = LiveOIBenchDataset(name_or_path=path).test_set + assert default is not None + assert [r["problem_id"] for r in default] == [BATCH_ID] + both = LiveOIBenchDataset(name_or_path=path, task_type=None).test_set + assert both is not None + assert len(both) == 2 + + +def test_year_and_competition_filters_apply(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID), _problem(DASHED_ID)]) + by_year = LiveOIBenchDataset(name_or_path=path, year="2024").test_set + assert by_year is not None + assert [r["problem_id"] for r in by_year] == [DASHED_ID] + by_competition = LiveOIBenchDataset(name_or_path=path, competition="IOI").test_set + assert by_competition is not None + assert [r["problem_id"] for r in by_competition] == [BATCH_ID] + + +def test_a_filter_matching_nothing_raises_instead_of_scoring_an_empty_set(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + with pytest.raises(ValueError, match="empty after filtering"): + LiveOIBenchDataset(name_or_path=path, year="1999") + + +def test_a_missing_test_corpus_names_the_command_that_builds_it(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)], with_tests=False) + with pytest.raises(FileNotFoundError, match="materialize_liveoibench_tests"): + LiveOIBenchDataset(name_or_path=path) + + +def test_prompts_can_be_loaded_without_the_corpus(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)], with_tests=False) + dataset = LiveOIBenchDataset(name_or_path=path, require_tests=False) + assert dataset.test_set is not None + assert len(dataset.test_set) == 1 + + +def test_an_external_tests_root_is_honoured(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)], with_tests=False) + external = tmp_path / "upstream_root" + case_dir = problem_tests_dir(str(external), BATCH_ID) + os.makedirs(case_dir) + dataset = LiveOIBenchDataset(name_or_path=path, tests_root=str(external)) + assert dataset.test_set is not None + assert list(dataset.test_set)[0]["tests_dir"] == case_dir + + +def test_a_problem_absent_from_the_test_parquet_is_a_loud_join_failure(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + # A second problem in the problems parquet only -- the two repos out of step. + problems_dir = tmp_path / "LiveOIBench" / "data" + pq.write_table( + pa.Table.from_pylist([_problem(BATCH_ID), _problem(DASHED_ID)]), + problems_dir / "liveoibench_v1.parquet", + ) + with pytest.raises(ValueError, match="out of step"): + LiveOIBenchDataset(name_or_path=path) + + +def test_a_null_subtasks_cell_is_the_same_loud_join_failure(tmp_path): + """A NULL rubric must not read as a present-but-empty one. + + Coerced to "{}" it passes the loader's own check and only surfaces later, + once per sample, at grading time -- where it looks like a scoring problem + rather than two parquets out of step. + """ + path = _stage(tmp_path, [_problem(BATCH_ID)]) + tests_repo = tmp_path / "LiveOIBench_tests" + pq.write_table( + pa.Table.from_pylist([{"id": 0, "problem_id": BATCH_ID, "subtasks": None}]), + tests_repo / year_parquet_name("2025"), + ) + with pytest.raises(ValueError, match="out of step"): + LiveOIBenchDataset(name_or_path=path) + + +def test_a_missing_year_parquet_names_the_download_command(tmp_path): + path = _stage(tmp_path, [_problem(BATCH_ID)]) + os.remove(tmp_path / "LiveOIBench_tests" / year_parquet_name("2025")) + with pytest.raises(FileNotFoundError, match="sieval dataset download"): + LiveOIBenchDataset(name_or_path=path) diff --git a/tests/unit/scripts/test_materialize_liveoibench_tests.py b/tests/unit/scripts/test_materialize_liveoibench_tests.py new file mode 100644 index 00000000..bc8cc6b7 --- /dev/null +++ b/tests/unit/scripts/test_materialize_liveoibench_tests.py @@ -0,0 +1,93 @@ +"""Unit tests for unpacking the LiveOIBench test corpus. + +The load-bearing behaviour is atomicity. The unpack writes >30 GB and takes long +enough to be interrupted — Ctrl-C, a full disk, an OOM kill — and both the +script's own skip check and the dataset loader's ``require_tests`` guard read a +directory's *existence* as "this problem is materialized". A problem written in +place would therefore come back from a crash looking complete, and the missing +cases would surface much later as a subtask naming a test that is not there. + +AI-Generated Code - Claude Opus 5 (Anthropic) +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] + + +def _load_module(): + """Import the script by path; `scripts/` is not a package.""" + path = ROOT / "scripts" / "materialize_liveoibench_tests.py" + spec = importlib.util.spec_from_file_location("_materialize_liveoibench", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +materialize = _load_module() + +TESTS = { + "a1": {"input": "1\n", "output": "2\n"}, + "a2": {"input": "3\n", "output": "4\n"}, +} + + +def test_a_problems_cases_land_in_the_upstream_layout(tmp_path): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + materialize.write_problem_tests(out_dir, TESTS) + assert sorted(p.name for p in out_dir.iterdir()) == [ + "a1.in", + "a1.out", + "a2.in", + "a2.out", + ] + assert (out_dir / "a2.out").read_text() == "4\n" + + +def test_an_interrupted_problem_leaves_no_directory_to_mistake_for_a_whole_one( + tmp_path, monkeypatch +): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + + real_write = Path.write_text + state = {"n": 0} + + def exploding_write(self, data, *args, **kwargs): + state["n"] += 1 + if state["n"] > 2: + raise KeyboardInterrupt("simulated Ctrl-C mid-problem") + return real_write(self, data, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", exploding_write) + with pytest.raises(KeyboardInterrupt): + materialize.write_problem_tests(out_dir, TESTS) + + # The guard that matters: nothing named `tests` exists, so neither the + # script's skip check nor `require_tests` reads this problem as done. + assert not out_dir.exists() + assert out_dir.with_name("tests.partial").is_dir() + + +def test_a_crashed_partial_is_discarded_rather_than_merged(tmp_path): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + stale = out_dir.with_name("tests.partial") + stale.mkdir(parents=True) + (stale / "ghost.in").write_text("from an earlier crash\n") + + materialize.write_problem_tests(out_dir, TESTS) + assert not (out_dir / "ghost.in").exists() + assert not stale.exists() + + +def test_overwriting_replaces_the_whole_directory(tmp_path): + out_dir = tmp_path / "IOI" / "2025" / "contest" / "beechtree" / "tests" + materialize.write_problem_tests(out_dir, {"old": {"input": "x", "output": "y"}}) + materialize.write_problem_tests(out_dir, TESTS) + assert not (out_dir / "old.in").exists() + assert (out_dir / "a1.in").read_text() == "1\n" diff --git a/tests/unit/tasks/test_liveoibench_0shot_gen.py b/tests/unit/tasks/test_liveoibench_0shot_gen.py new file mode 100644 index 00000000..22cd89cb --- /dev/null +++ b/tests/unit/tasks/test_liveoibench_0shot_gen.py @@ -0,0 +1,749 @@ +"""Unit tests for the LiveOIBench 0-shot chat task. + +Three things are load-bearing here and none of them live in the evaluator: + +* what goes on the wire — the problem's own limits, the grader sources, and a + test *directory* rather than ~140 MB of inlined cases; +* mapping the returned verdict vector onto subtasks by the names the evaluator + reports, rather than by an ordering the task re-derives; +* best-of-n, which is how upstream reports a model — not mean, not pass@k. + +The execution guard itself is the vendored evaluator's, a separate deployable +with its own tests. + +AI-Generated Code - Claude Opus 4.5 (Anthropic) +""" + +import json +import os + +import pytest +from datasets import Dataset as HFDataset +from datasets import DatasetDict as HFDatasetDict + +from sieval.core.models import Request, Response +from sieval.core.models.chat_model import ChatModel +from sieval.core.tasks import ( + NonRetriableSampleError, + TaskContext, + build_prediction_record, +) +from sieval.datasets.liveoibench import LiveOIBenchDataset +from sieval.tasks.liveoibench_0shot_gen import LiveOIBenchZeroShotGenTask +from tests.conftest import HandlerTransport + +PROBLEM_ID = "IOI-2025-contest-beechtree" +# Two subtasks: the first pays 30 for both its tests, the second 70 for its one. +SUBTASKS = { + "1": {"task": "Subtask 1", "score": 30, "testcases": ["a1", "a2"]}, + "2": {"task": "Subtask 2", "score": 70, "testcases": ["b1"]}, +} +CASE_NAMES = ["a1", "a2", "b1"] + + +class _StubChatModel(ChatModel): + def __init__( + self, replies: tuple[str, ...] = ("```beechtree.cpp\nint main(){}\n```",) + ): + self._replies = replies + super().__init__(model="mock-chat", api_key="fake") + + def _build_default_transport(self) -> HandlerTransport: + return HandlerTransport(self._stub_arun, "openai_chat") + + async def _stub_arun(self, req: Request) -> Response: + return Response(texts=self._replies[: req.sampling.n]) + + +class _Response: + def __init__(self, payload: dict): + self._payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return self._payload + + +class _Evaluator: + """Stands in for the code-eval service, one scripted reply per rollout.""" + + def __init__(self, verdict_sets: list[list[bool]]): + self._verdict_sets = verdict_sets + self.bodies: list[dict] = [] + self.deadlines: list[float] = [] + + async def post(self, url, *, json, timeout): + _ = url + self.bodies.append(json) + self.deadlines.append(timeout) + verdicts = self._verdict_sets[len(self.bodies) - 1] + return _Response( + { + "status": all(verdicts), + "msg": "graded", + "data": { + "n_cases": len(verdicts), + "n_passed": sum(verdicts), + "case_verdicts": verdicts, + "case_names": CASE_NAMES, + "peak_memory_mb": 12.5, + }, + } + ) + + async def aclose(self) -> None: + return None + + +def _raw( + tests_dir: str = "/staged/IOI/2025/contest/beechtree/tests", **overrides +) -> dict: + return { + "problem_id": PROBLEM_ID, + "competition": "IOI", + "contest": "contest", + "task_name": "beechtree", + "problem_statement": "# Beech Tree\n", + "time_limit": 1.5, + "memory_limit": 2048.0, + "task_type": "batch", + "difficulty": 12, + "algorithms": '["tree"]', + "grader_code": "", + "starter_code": "", + "subtasks": json.dumps(SUBTASKS), + "contest_id": "IOI-2025-contest", + "tests_dir": tests_dir, + } | overrides + + +def _task(raw: dict, evaluator: _Evaluator | None = None, **kwargs): + dataset = LiveOIBenchDataset( + _hf_dict=HFDatasetDict({"test": HFDataset.from_list([raw])}) + ) + task = LiveOIBenchZeroShotGenTask(dataset, _StubChatModel(), **kwargs) + if evaluator is not None: + task._http_client = evaluator # the real client is never used + return task + + +async def _grade(raw: dict, predictions: list[str], verdict_sets, **kwargs): + evaluator = _Evaluator(verdict_sets) + task = _task(raw, evaluator, **kwargs) + try: + _, judgement = await task.feedback( + build_prediction_record(predictions), + TaskContext(sample_id=0, raw_sample=raw), + ) + finally: + await task.shutdown() + return judgement, evaluator + + +# --------------------------------------------------------------------------- # +# Prompt and extraction +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_the_prompt_is_a_single_user_turn(): + # Upstream's default run sends no system message. + task = _task(_raw()) + record = await task.preprocess(_raw(), TaskContext(sample_id=0)) + await task.shutdown() + assert [m["role"] for m in record["prompt"]] == ["user"] + assert "Time limit: 1.5 seconds" in record["prompt"][0]["content"] + assert "'```beechtree.cpp'" in record["prompt"][0]["content"] + + +@pytest.mark.anyio +async def test_extraction_prefers_the_tasks_own_fence(): + task = _task(_raw()) + reply = ( + "Here is a sketch:\n```cpp\nint wrong(){}\n```\n" + "And the answer:\n```beechtree.cpp\nint right(){}\n```\n" + ) + record = await task.postprocess( + Response(texts=(reply,)), TaskContext(sample_id=0, raw_sample=_raw()) + ) + await task.shutdown() + assert record["rollouts"][0]["prediction"] == "int right(){}" + + +@pytest.mark.anyio +async def test_a_reply_with_no_code_records_a_miss(): + task = _task(_raw()) + record = await task.postprocess( + Response(texts=("I cannot solve this.",)), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + await task.shutdown() + assert record["rollouts"][0].get("prediction") is None + assert record["rollouts"][0]["extracted"] is False + + +# --------------------------------------------------------------------------- # +# What goes on the wire +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_the_request_carries_the_problems_own_limits_and_a_test_directory(): + _, evaluator = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + (body,) = evaluator.bodies + assert body["source"] == "liveoibench" and body["lang"] == "cpp" + assert body["timeout_per_case"] == 1.5 + assert body["memory_limit"] == 2048 + assert body["entry_filename"] == "beechtree.cpp" + # A directory, not ~140 MB of inlined cases. + assert body["test_dir"].endswith("beechtree/tests") + assert "test" not in body + + +@pytest.mark.anyio +async def test_grader_sources_are_sent_for_the_problems_that_ship_them(): + raw = _raw( + grader_code=json.dumps( + {"grader.cpp": "int main(){}", "beechtree.h": "void f();"} + ) + ) + _, evaluator = await _grade(raw, ["void f(){}"], [[True, True, True]]) + assert evaluator.bodies[0]["files"] == { + "grader.cpp": "int main(){}", + "beechtree.h": "void f();", + } + + +@pytest.mark.anyio +async def test_inline_mode_ships_the_cases_read_from_disk(tmp_path): + for name, payload in (("a1", "1\n"), ("a2", "2\n"), ("b1", "3\n")): + (tmp_path / f"{name}.in").write_text(payload) + (tmp_path / f"{name}.out").write_text(payload) + raw = _raw(tests_dir=str(tmp_path)) + _, evaluator = await _grade( + raw, ["int main(){}"], [[True, True, True]], inline_tests=True + ) + body = evaluator.bodies[0] + assert "test_dir" not in body + assert body["test"]["names"] == CASE_NAMES + assert body["test"]["inputs"] == ["1\n", "2\n", "3\n"] + + +@pytest.mark.anyio +async def test_inline_mode_refuses_a_test_input_with_no_expected_output(tmp_path): + (tmp_path / "a1.in").write_text("1\n") + raw = _raw(tests_dir=str(tmp_path)) + with pytest.raises(NonRetriableSampleError, match="no .out"): + await _grade(raw, ["int main(){}"], [[True]], inline_tests=True) + + +@pytest.mark.anyio +async def test_the_http_deadline_outlasts_every_budget_the_evaluator_can_spend( + tmp_path, +): + for name in CASE_NAMES: + (tmp_path / f"{name}.in").write_text("1\n") + (tmp_path / f"{name}.out").write_text("1\n") + _, evaluator = await _grade( + _raw(tests_dir=str(tmp_path)), ["int main(){}"], [[True, True, True]] + ) + # 60s compile bound + 3 cases * (1.5 * 1.2 + 5) + 30s slack. + deadline = evaluator.deadlines[0] + assert deadline.read == pytest.approx(60.0 + 3 * 6.8 + 30.0) + # `pool` is deliberately unset: these deadlines differ by an order of + # magnitude across problems, and a bare float would make a small problem + # queued behind large ones fail while waiting for a connection. + assert deadline.pool is None + + +# --------------------------------------------------------------------------- # +# Verdicts to subtask scores +# --------------------------------------------------------------------------- # +@pytest.mark.anyio +async def test_verdicts_are_mapped_onto_subtasks_by_the_names_the_evaluator_reports(): + # b1 alone pays 70; a1/a2 are incomplete so subtask 1 pays nothing. + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, False, True]]) + assert judgement["score"] == 70.0 + assert judgement["metrics"]["relative_score"] == 70.0 + assert judgement["metrics"]["tests_passed_pct"] == pytest.approx(200 / 3) + assert judgement["rollouts"][0]["extra"]["subtasks"]["1"]["score"] == 0 + assert judgement["rollouts"][0]["extra"]["subtasks"]["2"]["score"] == 70 + + +@pytest.mark.anyio +async def test_a_full_solve_is_correct_and_aces(): + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + assert judgement["rollouts"][0]["correct"] is True + assert judgement["metrics"]["ace"] is True + assert judgement["score"] == 100.0 + + +@pytest.mark.anyio +async def test_the_judgement_records_no_reference_and_names_the_rubric(): + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + assert judgement["reference"] is None + assert judgement["extra"]["total_points"] == 100 + assert judgement["extra"]["n_subtasks"] == 2 + assert judgement["extra"]["problem_id"] == PROBLEM_ID + + +@pytest.mark.anyio +async def test_the_best_rollout_is_reported_not_the_first_or_the_mean(): + # Upstream samples n and reports the highest-scoring candidate. + judgement, _ = await _grade( + _raw(), + ["int a(){}", "int b(){}", "int c(){}"], + [[True, True, False], [True, False, True], [False, False, False]], + n=3, + ) + assert judgement["score"] == 70.0, "the 70-point rollout must win" + assert judgement["extra"]["best_rollout"] == 1 + assert judgement["n_rollouts"] == 3 + + +@pytest.mark.anyio +async def test_ties_on_score_are_broken_by_tests_passed(): + # Both score 0; the second passed more tests, so it is the better candidate. + judgement, _ = await _grade( + _raw(), + ["int a(){}", "int b(){}"], + [[False, False, False], [True, False, False]], + n=2, + ) + assert judgement["score"] == 0.0 + assert judgement["extra"]["best_rollout"] == 1 + + +@pytest.mark.anyio +async def test_an_evaluator_without_the_cpp_path_is_an_error_not_a_zero(): + # `status` alone cannot attribute subtasks, so scoring from it would report + # every partial solution as zero. + class _OldEvaluator(_Evaluator): + async def post(self, url, *, json, timeout): + self.bodies.append(json) + return _Response( + {"status": False, "msg": "not supported data source", "data": None} + ) + + task = _task(_raw(), _OldEvaluator([])) + try: + with pytest.raises(RuntimeError, match="no per-case verdicts"): + await task.feedback( + build_prediction_record(["int main(){}"]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_an_empty_test_directory_names_itself_instead_of_scoring_zero(): + """A half-written materialized problem would otherwise read as a submission + that failed every subtask, since an empty verdict vector scores nothing.""" + + class _NoCases(_Evaluator): + async def post(self, url, *, json, timeout): + self.bodies.append(json) + return _Response( + { + "status": False, + "msg": "no test cases to run", + "data": { + "n_cases": 0, + "n_passed": 0, + "case_verdicts": [], + "case_names": [], + "peak_memory_mb": 0.0, + }, + } + ) + + task = _task(_raw(), _NoCases([])) + try: + with pytest.raises(NonRetriableSampleError, match="no test cases"): + await task.feedback( + build_prediction_record(["int main(){}"]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_a_problem_with_no_rubric_fails_the_sample_without_retrying(): + task = _task(_raw(subtasks="{}")) + try: + with pytest.raises(NonRetriableSampleError, match="no subtasks"): + await task.feedback( + build_prediction_record(["int main(){}"]), + TaskContext(sample_id=0, raw_sample=_raw(subtasks="{}")), + ) + finally: + await task.shutdown() + + +# --------------------------------------------------------------------------- # +# Report +# --------------------------------------------------------------------------- # +class _Final: + """A finalized sample as `report` reads it: the dataset row it came from + (which names its contest), the judgement, and the prediction record + `health_metrics` counts extraction misses from.""" + + def __init__( + self, + judgement: dict, + predictions: list[str | None] | None = None, + raw: dict | None = None, + ): + self.raw_sample = raw if raw is not None else _raw() + self.feedback_result = judgement + self.postprocess_result = build_prediction_record( + predictions if predictions is not None else ["int main(){}"] + ) + + +class _Fail: + """A sample that never produced a judgement. It still carries its dataset + row, so it can depress the mean of the contest it belongs to.""" + + def __init__(self, raw: dict | None = None): + self.raw_sample = raw if raw is not None else _raw() + self.feedback_result = None + self.postprocess_result = None + + +def _contestant_table(path, rows: list[dict]): + import pyarrow as pa + import pyarrow.parquet as pq + + pq.write_table(pa.Table.from_pylist(rows), path) + return path + + +def _contest_row( + contest_id: str, problems: list[str], ranking: list[dict], **overrides +): + return { + "contest_id": contest_id, + "problems": json.dumps(problems), + "gold_cutoff": 90.0, + "silver_cutoff": 60.0, + "bronze_cutoff": 30.0, + "contestants_ranking": json.dumps(ranking), + } | overrides + + +@pytest.mark.anyio +async def test_the_report_averages_over_every_requested_sample(): + task = _task(_raw()) + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + try: + # One perfect problem plus one pipeline failure -> 50, not 100. Both are + # in the same contest, so the contest mean is the problem mean here. + report = await task.report([_Final(judgement)], [_Fail()]) + finally: + await task.shutdown() + assert report["relative_score"] == 50.0 + assert report["score"] == report["relative_score"] + assert report["score_key"] == "relative_score" + assert report["denominator_policy"] == "requested" + # Upstream's names: `pass_rate` counts problems fully solved, and a failed + # sample is not one of them. + assert report["pass_rate"] == 50.0 + assert report["tests_passed_pct"] == 50.0 + assert report["fails"] == 1.0 + assert report["n_problems"] == 2.0 + assert report["n_scored"] == 1.0 + + +@pytest.mark.anyio +async def test_relative_score_weights_contests_equally_not_problems(): + """Upstream means inside a contest, then across contests. + + A 1-problem contest and a 3-problem contest therefore weigh the same. Under + problem-weighting the perfect singleton would pull the headline to 25.0. + """ + solved, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + big = [ + _Fail(_raw(problem_id="IOI-2025-contest-a", task_name="a")), + _Fail(_raw(problem_id="IOI-2025-contest-b", task_name="b")), + _Fail(_raw(problem_id="IOI-2025-contest-c", task_name="c")), + ] + small = _Final( + solved, + raw=_raw(problem_id="APIO-2025-contest-x", contest_id="APIO-2025-contest"), + ) + task = _task(_raw()) + try: + report = await task.report([small], big) + finally: + await task.shutdown() + assert report["relative_score"] == 50.0 + assert report["n_contests"] == 2.0 + # `pass_rate` stays problem-level, which is where the two differ. + assert report["pass_rate"] == 25.0 + + +@pytest.mark.anyio +async def test_an_empty_run_still_declares_its_headline(): + task = _task(_raw()) + try: + report = await task.report([], [_Fail()]) + finally: + await task.shutdown() + assert report["score"] == 0.0 + assert report["score_key"] == "relative_score" + assert report["denominator_policy"] == "requested" + # The all-fail report carries the same columns as any other, so a reader + # does not KeyError on exactly the run that went worst. + for key in ("relative_score", "pass_rate", "tests_passed_pct", "n_problems"): + assert key in report + + +@pytest.mark.anyio +async def test_human_metrics_come_from_the_staged_contestant_table(tmp_path): + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "IOI-2025-contest", + [PROBLEM_ID], + [{"Rank": 1, "beechtree": 100}, {"Rank": 2, "beechtree": 40}], + ) + ], + ) + + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + task = _task(_raw()) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement)], []) + finally: + await task.shutdown() + + # 100 beats the 40 and ties the 100 -> 50th percentile, and clears gold. + assert report["human_percentile"] == 50.0 + assert report["gold_rate"] == 100.0 + assert report["n_contests"] == 1.0 + assert report["n_contests_ranked"] == 1.0 + + +@pytest.mark.anyio +async def test_usaco_problems_rank_against_their_division_row(tmp_path): + """The contestant table keys USACO by division, not by the round in the id. + + Without the routing every USACO problem misses the join and drops out of the + human metrics entirely — 132 of the 380 scored problems on the real data. + """ + platinum_id = "USACO-2025-January_Contest-platinum_Cowpatibility" + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "USACO-2025-January_Contest-platinum", + [platinum_id], + [], # every published USACO row ships cutoffs and no ranking + gold_cutoff=250.0, + silver_cutoff=150.0, + bronze_cutoff=50.0, + ) + ], + ) + raw = _raw( + problem_id=platinum_id, + task_name="platinum_Cowpatibility", + contest_id="USACO-2025-January_Contest", + ) + judgement, _ = await _grade(raw, ["int main(){}"], [[True, True, True]]) + task = _task(raw) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement, raw=raw)], []) + finally: + await task.shutdown() + + # Matched: the contest publishes no contestants, so there is no percentile, + # but the medal cutoffs still apply (upstream's `df.empty` branch). + assert "human_percentile" not in report + assert report["n_contests_ranked"] == 0.0 + assert report["n_contests_medalled"] == 1.0 + assert report["medal_rate"] == 100.0 + + +@pytest.mark.anyio +async def test_a_usaco_combined_contest_reports_neither_percentile_nor_medal(tmp_path): + """Upstream scores `-combined` from promotion thresholds that ship in its + own repository, not in the dataset, so it computes nothing here either.""" + bronze_id = "USACO-2025-January_Contest-bronze_Cowmpetition" + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "USACO-2025-January_Contest-combined", + [bronze_id], + [], + gold_cutoff=800.0, + silver_cutoff=750.0, + bronze_cutoff=700.0, + ) + ], + ) + raw = _raw( + problem_id=bronze_id, + task_name="bronze_Cowmpetition", + contest_id="USACO-2025-January_Contest", + ) + judgement, _ = await _grade(raw, ["int main(){}"], [[True, True, True]]) + task = _task(raw) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement, raw=raw)], []) + finally: + await task.shutdown() + + assert "human_percentile" not in report + assert "medal_rate" not in report + assert report["n_contests_medalled"] == 0.0 + # The problem is still scored; only its human comparison is unavailable. + assert report["relative_score"] == 100.0 + + +@pytest.mark.anyio +async def test_a_missing_contestant_table_leaves_the_scores_reportable(tmp_path): + judgement, _ = await _grade(_raw(), ["int main(){}"], [[True, True, True]]) + task = _task(_raw()) + task.dataset._contestants_path = str(tmp_path / "absent.parquet") + try: + report = await task.report([_Final(judgement)], []) + finally: + await task.shutdown() + assert report["relative_score"] == 100.0 + assert "human_percentile" not in report + # No table read at all, so not even the counts are reported: a 0 here would + # read as "measured, nothing ranked". + assert "n_contests_ranked" not in report + + +@pytest.mark.anyio +async def test_a_failed_sample_is_not_compared_against_the_human_field(tmp_path): + """A failed sample counts at zero in the score means, never in the ranking. + + Upstream re-totals the humans over exactly the tasks the model was *scored* + on, and never writes a problem result for one that produced no judgement. + Feeding it in penalises twice: the model total loses the points and the human + total gains that problem's column. Here that is the whole gap between the + 100th percentile and the 0th. + """ + solved = _raw() + failed = _raw(problem_id="IOI-2025-contest-sphinx", task_name="sphinx") + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "IOI-2025-contest", + [solved["problem_id"], failed["problem_id"]], + # Both humans are weak on `beechtree` and strong on `sphinx`. + [ + {"beechtree": 50, "sphinx": 100}, + {"beechtree": 30, "sphinx": 90}, + ], + ) + ], + ) + + judgement, _ = await _grade(solved, ["int main(){}"], [[True, True, True]]) + task = _task(solved) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([_Final(judgement)], [_Fail(failed)]) + finally: + await task.shutdown() + + # Scored on `beechtree` alone: 100 beats both 50 and 30 -> 100th percentile. + # Counting the failed `sphinx` at 0 would compare 100 against totals of + # 150/120 and report 0.0 instead. + assert report["human_percentile"] == 100.0 + assert report["n_contests_ranked"] == 1.0 + # The failure is still in the score means and the denominator. + assert report["relative_score"] == 50.0 + assert report["n_problems"] == 2.0 + + +@pytest.mark.anyio +async def test_a_contest_whose_problems_all_failed_reports_no_percentile(tmp_path): + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [_contest_row("IOI-2025-contest", [PROBLEM_ID], [{"beechtree": 50}])], + ) + task = _task(_raw()) + task.dataset._contestants_path = str(contestants) + try: + report = await task.report([], [_Fail()]) + finally: + await task.shutdown() + # Nothing was measured, so there is no percentile -- not a percentile of 0, + # which would read as "the model was beaten by the whole field". + assert "human_percentile" not in report + assert report["n_contests_ranked"] == 0.0 + + +@pytest.mark.anyio +async def test_two_problems_sharing_a_task_name_in_one_contest_are_announced( + tmp_path, +): + """Keying the model's scores by task name is what matches a human column. + + No published contest has two problems under one name (0 of 72), so this + cannot silently halve a contest without saying so. + """ + from loguru import logger + + messages: list[str] = [] + sink = logger.add(lambda m: messages.append(m.record["message"]), level="WARNING") + + twin_a = _raw(problem_id="IOI-2025-contest-dup1", task_name="samename") + twin_b = _raw(problem_id="IOI-2025-contest-dup2", task_name="samename") + contestants = _contestant_table( + tmp_path / "contest_results.parquet", + [ + _contest_row( + "IOI-2025-contest", + [twin_a["problem_id"], twin_b["problem_id"]], + [{"samename": 10}], + ) + ], + ) + judgement, _ = await _grade(twin_a, ["int main(){}"], [[True, True, True]]) + task = _task(twin_a) + task.dataset._contestants_path = str(contestants) + try: + await task.report( + [_Final(judgement, raw=twin_a), _Final(judgement, raw=twin_b)], [] + ) + finally: + await task.shutdown() + logger.remove(sink) + + assert any("distinct task name" in m for m in messages), messages + + +@pytest.mark.anyio +async def test_inference_with_no_rollouts_names_the_sample(): + """`next(...)` over no rollouts would raise StopIteration out of a coroutine, + which Python re-raises naming neither the sample nor the cause.""" + task = _task(_raw()) + try: + with pytest.raises(NonRetriableSampleError, match="no rollouts to grade"): + await task.feedback( + build_prediction_record([]), + TaskContext(sample_id=0, raw_sample=_raw()), + ) + finally: + await task.shutdown() + + +@pytest.mark.anyio +async def test_the_task_is_registered_under_its_file_name(): + from sieval.core.tasks.meta import get_task_class + + assert get_task_class("liveoibench_0shot_gen") is LiveOIBenchZeroShotGenTask + assert os.path.basename(__file__) == "test_liveoibench_0shot_gen.py" diff --git a/vendor/code-evaluator/VENDORED.md b/vendor/code-evaluator/VENDORED.md index 3b8b8a69..a26092b1 100644 --- a/vendor/code-evaluator/VENDORED.md +++ b/vendor/code-evaluator/VENDORED.md @@ -481,3 +481,73 @@ Two kinds, and the difference is a decision rather than a status: -H "Accept: application/vnd.oci.image.manifest.v1+json" \ "https://ghcr.io/v2/nuprl/agnostics/manifests/lua" | grep -i docker-content-digest ``` +- `app/exec_cpp.py`, `app/server.py` — **C++ execution**, for + `source="liveoibench"`. The first non-Python language the evaluator runs as + a *compiled* artifact: `g++ -std=gnu++17 -Wall -O2 -pipe -static -g`, then one + child per test case under `RLIMIT_CPU` / `RLIMIT_AS`. Both limits carry + upstream LiveOIBench's explicit 20% buffer, and a 10 ms poller kills a child + whose CPU time passes the buffered limit — the rule its `BatchJudge` applies, + ported rather than reinvented, along with its output comparison (strip; + single-number outputs compare at `rel_tol=abs_tol=1e-6`; else exact; else line + count plus stripped per-line match). + + Four differences from upstream LiveOIBench, all documented in the module: + compilation is bounded (`timeout`, default 60 s) where upstream's + `subprocess.run` has no timeout; every test always runs, since subtask scoring + needs the whole verdict vector; no checker path exists, because the published + dataset ships no `checkers/` directory, so upstream's own judge compares + outputs directly on this data; and a suite with **no** test case is refused + by name (`status=False`, `"no test cases to run"`) rather than graded. That + last one is a deployment fault — a half-written materialized directory — and + the two readings it otherwise gets are both wrong: an empty verdict vector + scores every subtask at zero, and the failure message has no first failure to + name, which raised `StopIteration` out of a coroutine. + + `Sample` gains `files` (extra sources compiled alongside — `grader.cpp`, + `{task}.h`) and `entry_filename`; the test model gains `names`, used only in + the failure message. `ResourceMetrics` gains `case_verdicts` — one bool per + case, in request order — because an olympiad subtask scores on its own test + group, so `n_passed` alone cannot be attributed. All fields are optional and + the other sources are untouched. + + The test model is still named `LiveCodeBenchTest` though two sources now share + it; renaming it would widen the diff against upstream without changing the + wire format. + + **`test_dir` is a trusted-caller field**, the only one here that steers the + server at an arbitrary host path. Not a read primitive — only `{name}.in` / + `{name}.out` are opened and their contents reach the child's stdin rather than + the response — but it assumes the trust boundary the `agnostics` entry above + spells out: in-cluster callers, a mounted corpus, the service not exposed. A + deployment that cannot hold that should send `inline_tests=True` instead and + reject `test_dir` at the ingress. + + **Deploy `docker/Dockerfile.multipl-e`** — there is no separate image for this + source. The base image has no toolchain, and a `-static` link additionally + needs `libstdc++-*-dev` and `libc6-dev`, but `apt-get install g++` already + pulls both: Debian's `g++` metapackage depends on the versioned compiler, which + depends on the matching `libstdc++-N-dev`, which depends on `libc6-dev`. + Measured inside the real base rather than reasoned about — on the current + `python:3.10-slim` (Debian 13.6), `g++` alone brings `libc6-dev 2.41` and + `libstdc++-14-dev`, `libstdc++.a` and `libc.a` both resolve, and the judge's + exact link (`g++ -std=gnu++17 -Wall -O2 -pipe -static -g`) builds and runs a + `bits/stdc++.h` program. + + An earlier `Dockerfile.cpp` named those two packages explicitly and was + otherwise byte-identical to `Dockerfile.multipl-e` minus its other toolchains. + It was removed: it pinned `libstdc++-12-dev` while the base has since moved to + gcc 14, so it installed a dev tree the compiler no longer uses — a second image + to maintain that was both redundant and drifting. + + Verified against g++ 14.2 on Debian: correct, partial, TLE, MLE, + compile-error, float-tolerance and grader-linked submissions all produce the + expected verdict vectors. Not yet upstream — land in `scitix/code-evaluator` + and re-vendor. + + **Not the same C++ path as `exec_lang`'s `cpp` row above**, and the two are not + merge candidates. That one is direct-run: one program, one all-or-nothing + verdict, budgets from the table. This one compiles against the problem's own + grader, runs one child *per official test case* under that problem's + `RLIMIT_CPU` / `RLIMIT_AS`, and returns the whole verdict vector — which is the + only shape IOI subtask scoring can be computed from. They are reached by + different `source` values and share no code. diff --git a/vendor/code-evaluator/app/exec_cpp.py b/vendor/code-evaluator/app/exec_cpp.py new file mode 100644 index 00000000..3d21efff --- /dev/null +++ b/vendor/code-evaluator/app/exec_cpp.py @@ -0,0 +1,415 @@ +"""C++ compile-and-run path, for LiveOIBench-style olympiad judging. + +Mirrors the reference judge of +https://github.com/LiveOIBench/LiveOIBench-Evaluation/blob/7759e3b8672307cfbdc8ab8e679bd87cc1dd4c12/src/judges/batch_judge.py +(``BatchJudge.run_test_case`` / ``BaseJudge.compile_cpp``), which is the only +place a LiveOIBench verdict is defined: + +* compile with ``g++ -std=gnu++17 -Wall -O2 -pipe -static -g``, the problem's + ``grader.cpp`` first on the command line when it has one; +* per test case, ``RLIMIT_CPU = ceil(time_limit * 1.2)`` and + ``RLIMIT_AS = ceil(memory_limit_mb * 1.2) MB`` -- upstream's explicit 20% + buffer -- with a poller that kills the process once its CPU time passes the + same buffered limit; +* a non-zero exit is a failure; otherwise stdout is compared against the + expected output by ``_compare_outputs`` below. + +Two deliberate differences, both bounds rather than rules: + +* **Compilation is bounded** (``compile_timeout``, default 60 s). Upstream calls + ``subprocess.run`` with no timeout, so a submission whose template expansion + never terminates wedges the judge. Nothing legitimate approaches the bound -- + a solution that cannot compile in a minute at ``-O2`` also cannot be measured. +* **Every test runs.** Upstream's ``stop_on_failure`` mode short-circuits a + subtask at its first failure; here the full verdict vector always comes back, + because subtask scoring needs it and a truncated vector silently scores every + partially-correct submission as zero. Upstream's default is also to run them + all. + +Unlike the LiveCodeBench path, this one executes a *compiled binary* rather than +interpreting a string, so the submission never shares an address space with the +evaluator. +""" + +import asyncio +import math +import os +import re +import resource +import subprocess +import tempfile +import threading +import time +from dataclasses import dataclass + +import psutil +from loguru import logger + +from .resource_monitor import ResourceStats + +COMPILE_COMMAND = ["g++", "-std=gnu++17", "-Wall", "-O2", "-pipe", "-static", "-g"] + +# Upstream's per-test wall clock: `process.communicate(input=..., timeout=120)`. +# The real budget is RLIMIT_CPU; this only catches a process that is not burning +# CPU at all (blocked on a read, sleeping). +WALL_TIMEOUT = 120.0 + +# Upstream's `cpu_time_limit *= 1.2` / `memory_limit_mb *= 1.2`. +LIMIT_BUFFER = 1.2 + +_NUMBER_RE = re.compile(r"^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$") +_LEADING_NUMBER_RE = re.compile(r"^[+\-]?(\d+(\.\d*)?|\.\d+)([eE][+\-]?\d+)?") + + +@dataclass +class CaseResult: + correct: bool + detail: str + cpu_time: float + memory_mb: float + + +@dataclass +class TestCase: + """One test case, either inline or on disk. + + A LiveOIBench problem averages ~140 MB of test data, so a run that shipped + every case inline would move the whole 33.5 GB corpus over HTTP once per + rollout. When the evaluator can see the materialized test tree, the caller + sends the directory instead and each case is read only while it runs. + """ + + name: str + input_data: str | None = None + expected: str | None = None + input_path: str | None = None + expected_path: str | None = None + + def read_input(self) -> str: + if self.input_path is not None: + with open(self.input_path, encoding="utf-8", errors="replace") as f: + return f.read() + return self.input_data or "" + + def read_expected(self) -> str: + if self.expected_path is not None: + with open(self.expected_path, encoding="utf-8", errors="replace") as f: + return f.read() + return self.expected or "" + + +def read_test_dir(test_dir: str) -> list[TestCase]: + """Collect ``{name}.in`` / ``{name}.out`` pairs from a materialized problem. + + Upstream (``Problem.get_test_inputs`` / ``get_test_outputs``) sorts the two + globs independently and pairs them by position, which silently misaligns + every later case if one ``.out`` is missing. Pairing by stem instead makes + that a loud error; on well-formed data the two agree, since sorting by name + *is* sorting by stem. + """ + if not os.path.isdir(test_dir): + raise FileNotFoundError(f"test directory not found: {test_dir}") + + entries = os.listdir(test_dir) + inputs = sorted(name[: -len(".in")] for name in entries if name.endswith(".in")) + outputs = {name[: -len(".out")] for name in entries if name.endswith(".out")} + + missing = [stem for stem in inputs if stem not in outputs] + if missing: + raise FileNotFoundError( + f"{len(missing)} test input(s) in {test_dir} have no matching .out " + f"(e.g. {missing[0]!r})" + ) + return [ + TestCase( + name=stem, + input_path=os.path.join(test_dir, f"{stem}.in"), + expected_path=os.path.join(test_dir, f"{stem}.out"), + ) + for stem in inputs + ] + + +def _is_single_number(text: str) -> bool: + """Upstream ``BatchJudge._is_single_number``.""" + lines = text.splitlines() + while lines and lines[-1].strip() == "": + lines.pop() + if len(lines) != 1: + return False + return bool(_NUMBER_RE.fullmatch(lines[0].strip())) + + +def _compare_numbers(gold: str, output: str) -> bool: + """Upstream ``BatchJudge._compare_numbers``.""" + m_gold = _LEADING_NUMBER_RE.match(gold.strip()) + m_out = _LEADING_NUMBER_RE.match(output.strip()) + if m_gold and m_out: + return math.isclose( + float(m_gold.group(0)), float(m_out.group(0)), rel_tol=1e-6, abs_tol=1e-6 + ) + return False + + +def _compare_outputs(gold_output: str, stdout_decoded: str) -> bool: + """Upstream ``BatchJudge._compare_outputs``. + + Both sides stripped; a single number on both sides compares with tolerance; + then exact match; then line count plus a stripped per-line match. + """ + gold_output = gold_output.strip() + stdout_decoded = stdout_decoded.strip() + + if _is_single_number(gold_output) and _is_single_number(stdout_decoded): + return _compare_numbers(gold_output, stdout_decoded) + + if gold_output == stdout_decoded: + return True + + gold_lines = gold_output.splitlines() + output_lines = stdout_decoded.splitlines() + if len(gold_lines) != len(output_lines): + return False + for gold_line, output_line in zip(gold_lines, output_lines): + if gold_line.strip() != output_line.strip(): + return False + return True + + +def _set_limits(cpu_time_limit: int, memory_limit_bytes: int): + """Upstream ``set_limits``, applied in the child before exec.""" + resource.setrlimit(resource.RLIMIT_CPU, (cpu_time_limit, cpu_time_limit)) + resource.setrlimit(resource.RLIMIT_AS, (memory_limit_bytes, memory_limit_bytes)) + + +def _monitor_process(proc: psutil.Process, time_limit: float, monitor_data: dict, interval=0.01): + """Upstream ``monitor_process``: sample CPU/RSS, kill once CPU passes the limit.""" + while proc.is_running(): + try: + cpu_time = proc.cpu_times().user + proc.cpu_times().system + mem_usage = proc.memory_info().rss / (1024 * 1024) + monitor_data["max_cpu_time"] = max(monitor_data["max_cpu_time"], cpu_time) + monitor_data["max_memory"] = max(monitor_data["max_memory"], mem_usage) + if cpu_time > time_limit: + proc.kill() + break + except psutil.NoSuchProcess: + break + time.sleep(interval) + + +def _compile( + workdir: str, + entry_filename: str, + grader_filename: str | None, + compile_timeout: float, +) -> tuple[bool, str]: + """Upstream ``BaseJudge.compile_cpp``, run inside the submission's workdir. + + ``cwd=workdir`` is what makes ``#include "task.h"`` resolve, the same way the + upstream judge's copied-together working directory does. + """ + sources = [grader_filename, entry_filename] if grader_filename else [entry_filename] + command = [*COMPILE_COMMAND, "-o", "solution", *sources] + try: + result = subprocess.run( + command, + cwd=workdir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=compile_timeout, + check=True, + ) + except subprocess.CalledProcessError as e: + return False, e.stderr or e.stdout or "compilation failed" + except subprocess.TimeoutExpired: + return False, f"compilation timed out after {compile_timeout}s" + except FileNotFoundError: + return False, "g++ not found: this evaluator image has no C++ toolchain" + return True, result.stdout + + +def _run_one_test( + workdir: str, + case: TestCase, + cpu_time_limit: float, + memory_limit_mb: float, +) -> CaseResult: + """One test case, with upstream's buffered limits and comparison.""" + input_data = case.read_input() + # Upstream buffers both limits by 20% before applying them. + cpu_time_limit = cpu_time_limit * LIMIT_BUFFER + memory_limit_mb = memory_limit_mb * LIMIT_BUFFER + + executable = os.path.join(workdir, "solution") + + def preexec(): + _set_limits( + int(math.ceil(cpu_time_limit)), + int(math.ceil(memory_limit_mb * 1024 * 1024)), + ) + + process = subprocess.Popen( + [executable], + cwd=workdir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=preexec, + ) + + monitor_data = {"max_cpu_time": 0.0, "max_memory": 0.0} + monitor_thread = None + try: + proc = psutil.Process(process.pid) + monitor_thread = threading.Thread( + target=_monitor_process, args=(proc, cpu_time_limit, monitor_data), daemon=True + ) + monitor_thread.start() + except psutil.NoSuchProcess: + # Exited before we could attach; the limits in the child still applied. + pass + + try: + stdout_data, stderr_data = process.communicate( + input=input_data.encode(), timeout=WALL_TIMEOUT + ) + except subprocess.TimeoutExpired: + process.kill() + stdout_data, stderr_data = process.communicate() + if monitor_thread is not None: + monitor_thread.join() + + cpu_time = monitor_data["max_cpu_time"] + memory_mb = monitor_data["max_memory"] + + if process.returncode == 0: + stdout_decoded = (stdout_data.decode(errors="replace") if stdout_data else "").strip() + correct = _compare_outputs(case.read_expected(), stdout_decoded) + return CaseResult(correct, "" if correct else "wrong answer", cpu_time, memory_mb) + + # A non-zero exit is a failure regardless of cause; the cause is reported + # only as free text, because upstream's judge does not classify it either -- + # `ResultType` exists in its tree but `BatchJudge` never assigns one. + if cpu_time >= cpu_time_limit: + detail = f"time limit exceeded ({cpu_time:.2f}s of {cpu_time_limit:.2f}s)" + elif process.returncode == -9: + detail = f"killed (peak rss {memory_mb:.0f}MB of {memory_limit_mb:.0f}MB)" + else: + stderr_text = (stderr_data.decode(errors="replace") if stderr_data else "").strip() + detail = f"runtime error [exit {process.returncode}]: {stderr_text[:200]}" + return CaseResult(False, detail, cpu_time, memory_mb) + + +def build_cases( + inputs: list[str] | None, + expect_outputs: list[str] | None, + names: list[str] | None = None, + test_dir: str | None = None, +) -> list[TestCase]: + """Normalize the two ways a caller can supply tests into one case list.""" + if test_dir: + return read_test_dir(test_dir) + inputs = inputs or [] + expect_outputs = expect_outputs or [] + return [ + TestCase( + name=names[i] if names and i < len(names) else f"#{i}", + input_data=input_data, + expected=expected, + ) + for i, (input_data, expected) in enumerate( + zip(inputs, expect_outputs, strict=True) + ) + ] + + +async def execute_tests( + code: str, + cases: list[TestCase], + entry_filename: str = "solution.cpp", + files: dict[str, str] | None = None, + timeout_per_case: float = 1.0, + memory_limit: int = 1024, + compile_timeout: float = 60.0, + max_workers: int = 4, +) -> tuple[bool, str, ResourceStats, int, list[bool]]: + """Compile *code* and run it against every case. + + Returns ``(all_passed, msg, stats, n_passed, verdicts)``. ``verdicts`` is one + bool per test **in case order** -- the caller maps them onto subtasks by + index, so the list is always as long as *cases*, including on a compilation + error (all ``False``). + + *files* are extra sources written beside the submission: the problem's + ``grader.cpp`` and ``{task}.h``. A ``grader.cpp`` among them is compiled with + the submission, which is how upstream grades the 41 problems that ship one. + + *max_workers* only bounds host load: verdicts are decided by each child's own + CPU-time and address-space limits, so they do not depend on how many run at + once. + """ + files = dict(files or {}) + stats = ResourceStats() + + if not cases: + # A problem with no test case is a deployment fault -- an empty or + # half-written materialized directory -- not a verdict on the submission. + # Said here rather than left to the failure message below, which would + # otherwise look for a first failure among no results and raise. + return False, "no test cases to run", stats, 0, [] + + with tempfile.TemporaryDirectory(prefix="liveoibench-") as workdir: + for filename, contents in files.items(): + # Flat by construction: upstream's bundles are `grader.cpp` / `task.h`. + with open(os.path.join(workdir, os.path.basename(filename)), "w") as f: + f.write(contents) + with open(os.path.join(workdir, entry_filename), "w") as f: + f.write(code) + + grader_filename = "grader.cpp" if "grader.cpp" in files else None + compiled, compile_msg = await asyncio.to_thread( + _compile, workdir, entry_filename, grader_filename, compile_timeout + ) + if not compiled: + logger.debug(f"compilation failed:\n{compile_msg}") + return ( + False, + f"compilation error: {compile_msg.strip()[:500]}", + stats, + 0, + [False] * len(cases), + ) + + semaphore = asyncio.Semaphore(max(1, max_workers)) + + async def run_case(case: TestCase) -> CaseResult: + async with semaphore: + return await asyncio.to_thread( + _run_one_test, workdir, case, timeout_per_case, memory_limit + ) + + results: list[CaseResult] = list( + await asyncio.gather(*(run_case(case) for case in cases)) + ) + + verdicts = [result.correct for result in results] + n_passed = sum(verdicts) + all_passed = n_passed == len(verdicts) and bool(verdicts) + + if results: + stats.peak_memory_mb = max(result.memory_mb for result in results) + stats.memory_mb = sum(result.memory_mb for result in results) / len(results) + + if all_passed: + msg = f"passed {n_passed}/{len(verdicts)} test cases" + else: + first_failure = next(i for i, result in enumerate(results) if not result.correct) + msg = ( + f"passed {n_passed}/{len(verdicts)} test cases; " + f"first failure at test {cases[first_failure].name}: " + f"{results[first_failure].detail}" + ) + + return all_passed, msg, stats, n_passed, verdicts diff --git a/vendor/code-evaluator/app/server.py b/vendor/code-evaluator/app/server.py index 2ff0b866..33dd0adc 100644 --- a/vendor/code-evaluator/app/server.py +++ b/vendor/code-evaluator/app/server.py @@ -10,6 +10,8 @@ from pydantic import BaseModel from .exec_agnostics import execute_agnostics +from .exec_cpp import build_cases +from .exec_cpp import execute_tests as exec_cpp_test from .exec_js import execute_code as exec_js from .exec_lang import LANGUAGES, toolchain_entry, toolchain_present from .exec_lang import execute_code as exec_lang @@ -74,6 +76,18 @@ class ResourceMetrics(BaseModel): ``None`` only when nothing ran at all, i.e. an unsupported language or source, where ``data`` itself is ``None``. + ``case_verdicts`` is the per-case detail behind ``n_passed``: one bool per + case, in request order. Only ``liveoibench`` fills it, because only it needs + *which* cases passed -- an olympiad subtask scores on its own test group, so + a count cannot be attributed. ``None`` elsewhere means "not reported", which + is why the count fields stay rather than being derived from it. + + ``case_names`` is what those verdicts belong to, positionally. It matters + when the caller passed ``test_dir`` rather than inline cases: the server + decided the order by listing that directory, and a caller that re-derived it + would be keeping a second copy of the ordering rule -- the kind that agrees + until one side changes. + Kept as one flat model rather than a per-source subclass: FastAPI filters the response against this route's declared model, so a subclass returned from one branch would have its extra fields silently stripped. @@ -85,6 +99,8 @@ class ResourceMetrics(BaseModel): peak_memory_mb: float n_cases: int | None = None n_passed: int | None = None + case_verdicts: list[bool] | None = None + case_names: list[str] | None = None # quotebench only. They live on this flat model for the reason the docstring # above gives: a per-source subclass would have these stripped by FastAPI's # response filtering. `error_class` is the failure taxonomy @@ -270,6 +286,10 @@ class LiveCodeBenchTest(BaseModel): inputs: list[str] outputs: list[str] fn_name: str | None = None + # `liveoibench` only, and only for the failure message: its test cases have + # official names ("boi1a") that a caller reading the log wants back. The + # verdict-to-subtask mapping is positional, so nothing depends on it. + names: list[str] | None = None class Sample(BaseModel): @@ -279,6 +299,16 @@ class Sample(BaseModel): test: LiveCodeBenchTest | None = None lang: str = "python" timeout: float | None = None + # `liveoibench` only: extra sources written beside the submission before + # compiling -- the problem's `grader.cpp` and `{task}.h` -- and the filename + # the submission itself is written as, which its `#include` has to match. + files: dict[str, str] | None = None + entry_filename: str | None = None + # `liveoibench` only: a directory of `{name}.in` / `{name}.out` pairs this + # evaluator can read, used instead of `test`. A problem averages ~140 MB of + # test data, so inlining it would move the corpus over HTTP once per rollout; + # a caller whose evaluator cannot see the volume sends `test` instead. + test_dir: str | None = None # Per-case budget, applied on top of `timeout`, which stays a whole-suite wall. # Opt-in: absent keeps the previous behaviour exactly. Official LiveCodeBench # budgets per case rather than per suite -- see `exec_py_test.execute_test`. @@ -544,6 +574,94 @@ async def evaluate(sample: Sample) -> BasicResponse[ResourceMetrics]: verifier_image=verifier_image, ), ) + elif sample.source == "liveoibench": + # 'liveoibench': compile the C++ submission once, then run every official + # test case. Unlike the livecodebench path this reports per-case verdicts + # and never stops early -- an olympiad subtask is scored on its own test + # group, so a submission that fails case 0 and passes the rest still + # earns points. + if sample.lang != "cpp": + return BasicResponse( + status=False, msg=f"not supported language: {sample.lang}", data=None + ) + if sample.test is None and not sample.test_dir: + return BasicResponse( + status=False, + msg="'liveoibench' requires either 'test' or 'test_dir'", + data=None, + ) + if ( + sample.test is not None + and not sample.test_dir + and len(sample.test.inputs) != len(sample.test.outputs) + ): + return BasicResponse( + status=False, + msg=( + f"test inputs/outputs length mismatch: " + f"{len(sample.test.inputs)} != {len(sample.test.outputs)}" + ), + data=None, + ) + + try: + cases = build_cases( + inputs=sample.test.inputs if sample.test else None, + expect_outputs=sample.test.outputs if sample.test else None, + names=sample.test.names if sample.test else None, + test_dir=sample.test_dir, + ) + except (FileNotFoundError, OSError) as e: + # An unreadable `test_dir` is a deployment problem (the evaluator + # cannot see the test volume), not a verdict on the submission. + logger.error(f"cannot read tests for sample '{sample.uuid}': {e}") + return BasicResponse(status=False, msg=f"cannot read tests: {e}", data=None) + + # The problem's own time limit, in seconds, per case; upstream buffers it + # by 20% inside the runner. `timeout` bounds compilation instead -- the + # one step with no per-case budget. + timeout_per_case = ( + sample.timeout_per_case if sample.timeout_per_case is not None else 1.0 + ) + compile_timeout = sample.timeout if sample.timeout is not None else 60.0 + + ok, msg, stats, n_passed, verdicts = await exec_cpp_test( + code=sample.code, + cases=cases, + entry_filename=sample.entry_filename or "solution.cpp", + files=sample.files, + timeout_per_case=timeout_per_case, + memory_limit=sample.memory_limit, + compile_timeout=compile_timeout, + ) + n_cases = len(cases) + + logger.info( + f"evaluate sample '{sample.uuid}' from '{sample.source}', " + f"language: {sample.lang}, timeout_per_case: {timeout_per_case}, " + f"compile_timeout: {compile_timeout}, memory_limit: {sample.memory_limit}, " + f"kwargs: {sample.kwargs}, status: {ok}, msg: {msg}, " + f"cases: {n_passed}/{n_cases}, " + f"avg_memory: {stats.memory_mb:.2f}MB, " + f"peak_memory: {stats.peak_memory_mb:.2f}MB" + ) + return BasicResponse( + status=ok, + msg=msg, + data=ResourceMetrics( + # Both CPU fields stay 0.0 here by construction: the per-case + # poller measures each child's CPU *seconds* to enforce + # RLIMIT_CPU, which is not this field's whole-service percentage. + avg_cpu_percent=stats.cpu_percent, + peak_cpu_percent=stats.peak_cpu_percent, + avg_memory_mb=stats.memory_mb, + peak_memory_mb=stats.peak_memory_mb, + n_cases=n_cases, + n_passed=n_passed, + case_verdicts=verdicts, + case_names=[case.name for case in cases], + ), + ) else: logger.error(f"not supported data source: {sample.source}") return BasicResponse( diff --git a/vendor/code-evaluator/docker/Dockerfile.multipl-e b/vendor/code-evaluator/docker/Dockerfile.multipl-e index 60f5bf89..8a4e90fc 100644 --- a/vendor/code-evaluator/docker/Dockerfile.multipl-e +++ b/vendor/code-evaluator/docker/Dockerfile.multipl-e @@ -7,6 +7,13 @@ FROM python:3.10-slim # # g++ -- humaneval-cpp / mbpp-cpp (build step; `bits/stdc++.h`, which # every cpp prompt includes, is GCC's) +# AND source="liveoibench" (app/exec_cpp.py, a different C++ path: +# per-test-case, `-static`, under the +# problem's own RLIMIT_CPU/RLIMIT_AS. +# It needs `libstdc++-*-dev` + +# `libc6-dev` to link statically, which +# `g++` already depends on -- verified in +# this base, do not "simplify" g++ away) # bash -- humaneval-sh / mbpp-sh (in the base image; named here so a # base change that drops it is visible) # perl -- humaneval-pl / mbpp-pl