diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/events.jsonl b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/events.jsonl new file mode 100644 index 0000000000..5956a40e03 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/events.jsonl @@ -0,0 +1 @@ +{"details": {"lock_sha256": "9aeeaa4987227f5ac4761349d0979ef6b425cb724fb54d1fe21b89f9cb62e678"}, "event": "freeze_locked", "phase": "freeze", "previous_event_sha256": null, "schema_version": 1, "sequence": 1, "sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", "time_utc": "2026-08-01T03:19:38.915424Z"} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/fetch_authority.py b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/fetch_authority.py new file mode 100644 index 0000000000..121d368499 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/fetch_authority.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Fetch or verify the exact official-document pages allowed by this run.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import html +import io +import re +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + + +RUN = Path(__file__).resolve().parent +ALLOWLISTS = RUN / "freeze" / "allowlists" +MANIFEST = RUN / "freeze" / "authority-manifest.tsv" +MODES = ("S", "C", "X", "Q", "W", "M", "R", "K") +FIELDS = ( + "mode", + "requested_url", + "fetch_url", + "final_url", + "status", + "content_type", + "bytes", + "sha256", + "fragment_found", + "retrieved_utc", +) + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def read_pinned(path: Path) -> bytes: + data = path.read_bytes() + if (RUN / "freeze" / "LOCK.json").exists(): + import protocol + + if sha256(data) != protocol.frozen_file_digest(path): + raise ValueError(f"frozen authority input changed while being read: {path}") + return data + + +def load_urls() -> list[tuple[str, str]]: + rows: list[tuple[str, str]] = [] + for mode in MODES: + path = ALLOWLISTS / f"{mode}.txt" + for url in read_pinned(path).decode().splitlines(): + if not re.fullmatch(r"https://doc\.rust-lang\.org/\S+", url): + raise ValueError(f"invalid allowlisted URL: {url!r}") + rows.append((mode, url)) + return rows + + +def without_fragment(url: str) -> str: + parts = urllib.parse.urlsplit(url) + return urllib.parse.urlunsplit((parts.scheme, parts.netloc, parts.path, parts.query, "")) + + +def fragment_present(requested_url: str, body: bytes) -> bool: + fragment = urllib.parse.unquote(urllib.parse.urlsplit(requested_url).fragment) + if not fragment: + return True + text = body.decode("utf-8", errors="replace") + escaped = html.escape(fragment, quote=True) + patterns = ( + rf'\bid=["\']{re.escape(fragment)}["\']', + rf'\bid=["\']{re.escape(escaped)}["\']', + rf'\bname=["\']{re.escape(fragment)}["\']', + ) + return any(re.search(pattern, text) for pattern in patterns) + + +def fetch(url: str) -> dict[str, object]: + request = urllib.request.Request( + url, + headers={"User-Agent": "unsafe-rust-skill-evaluation/1 authority-freeze"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read() + return { + "final_url": response.geturl(), + "status": response.status, + "content_type": response.headers.get_content_type(), + "bytes": len(body), + "sha256": sha256(body), + "body": body, + "retrieved_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + } + + +def build_rows() -> list[dict[str, str]]: + requested = load_urls() + fetched: dict[str, dict[str, object]] = {} + for _mode, requested_url in requested: + fetch_url = without_fragment(requested_url) + if fetch_url not in fetched: + fetched[fetch_url] = fetch(fetch_url) + + rows: list[dict[str, str]] = [] + for mode, requested_url in requested: + fetch_url = without_fragment(requested_url) + result = fetched[fetch_url] + found = fragment_present(requested_url, result["body"]) + if not found: + raise ValueError(f"fragment not present in retrieved page: {requested_url}") + rows.append( + { + "mode": mode, + "requested_url": requested_url, + "fetch_url": fetch_url, + "final_url": str(result["final_url"]), + "status": str(result["status"]), + "content_type": str(result["content_type"]), + "bytes": str(result["bytes"]), + "sha256": str(result["sha256"]), + "fragment_found": "true", + "retrieved_utc": str(result["retrieved_utc"]), + } + ) + return rows + + +def render(rows: list[dict[str, str]]) -> str: + output = io.StringIO(newline="") + writer = csv.DictWriter(output, fieldnames=FIELDS, dialect="excel-tab", lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return output.getvalue() + + +def read_manifest() -> list[dict[str, str]]: + rows = list( + csv.DictReader(io.StringIO(read_pinned(MANIFEST).decode()), dialect="excel-tab") + ) + if not rows or tuple(rows[0]) != FIELDS: + raise ValueError("authority manifest has unexpected columns") + return rows + + +def verify() -> None: + rows = read_manifest() + expected_pairs = load_urls() + actual_pairs = [(row["mode"], row["requested_url"]) for row in rows] + if actual_pairs != expected_pairs: + raise ValueError("authority manifest does not match allowlist ordering") + + fetched: dict[str, dict[str, object]] = {} + for row in rows: + fetch_url = row["fetch_url"] + if fetch_url != without_fragment(row["requested_url"]): + raise ValueError(f"incorrect fetch URL for {row['requested_url']}") + if fetch_url not in fetched: + fetched[fetch_url] = fetch(fetch_url) + result = fetched[fetch_url] + checks = { + "final_url": str(result["final_url"]), + "status": str(result["status"]), + "content_type": str(result["content_type"]), + "bytes": str(result["bytes"]), + "sha256": str(result["sha256"]), + } + for field, actual in checks.items(): + if row[field] != actual: + raise ValueError( + f"authority drift for {row['requested_url']}: " + f"{field} frozen={row[field]!r} live={actual!r}" + ) + if row["fragment_found"] != "true" or not fragment_present( + row["requested_url"], result["body"] + ): + raise ValueError(f"fragment missing for {row['requested_url']}") + print(f"verified {len(rows)} allowlist entries across {len(fetched)} pages") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--write", action="store_true", help="retrieve and write the frozen manifest") + parser.add_argument( + "--record-wave", + type=int, + choices=range(1, 6), + help="verify live bytes and append a successful verification event for this collection wave", + ) + args = parser.parse_args() + if args.write and args.record_wave is not None: + raise SystemExit("--write and --record-wave are mutually exclusive") + if args.write: + if (RUN / "freeze" / "LOCK.json").exists(): + raise SystemExit("refusing to rewrite authority manifest after freeze lock") + MANIFEST.write_text(render(build_rows())) + print(f"wrote {MANIFEST.relative_to(RUN)}") + else: + operation_lock_handle = None + if args.record_wave is not None: + import protocol + + operation_lock_handle = protocol.acquire_operation_lock() + protocol.validate_static(require_lock=True, announce=False) + protocol.assert_freeze_locked() + protocol.assert_authority_verification_allowed(args.record_wave) + verify() + if args.record_wave is not None: + protocol.append_event( + "collection", + "authority_verified", + digest=protocol.frozen_file_digest(MANIFEST), + details={"wave": args.record_wave, "entries": len(read_manifest())}, + ) + + +if __name__ == "__main__": + main() diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/LOCK.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/LOCK.json new file mode 100644 index 0000000000..08c149717a --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/LOCK.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "status": "FROZEN", + "file_manifest_sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", + "review_signoffs": [ + { + "reviewer_id": "codex-analyze-n-overall", + "verdict": "PASS/FREEZE", + "file_manifest_sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", + "scope": "Independent authentication and holistic review of all 47 manifest-listed bytes: protocol.py, prepare.py, fetch_authority.py, frozen plan/manifest/policies/prompts/schemas/randomization/allowlists/authority manifest/oracles/rubrics, sealed maps/seeds, V3 and V2 package identities, all eight target identities and source-oracle claims, and zero-report precollection state.", + "reviewed_utc": "2026-08-01T03:16:47Z" + }, + { + "reviewer_id": "v3-fixtures-domain-audit", + "verdict": "PASS/FREEZE", + "file_manifest_sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", + "scope": "Independent byte authentication of all 47 manifest-listed files; complete review of protocol lifecycle, freeze and failure semantics, policies, prompts, schemas, randomization and sealed maps, blinding, authority allowlists and oracle/rubric semantics, V3 and V2 package identities, all eight fixture and target identities, and zero-report precollection state. The 76 allowlist entries across 46 live official pages were also reverified against frozen hashes without recording an event.", + "reviewed_utc": "2026-08-01T03:18:56Z" + } + ], + "reports_collected_before_lock": 0, + "locked_utc": "2026-08-01T03:19:11Z" +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/C.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/C.txt new file mode 100644 index 0000000000..b65efda7ef --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/C.txt @@ -0,0 +1,18 @@ +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_or +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_or +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_or +https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html +https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html +https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/K.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/K.txt new file mode 100644 index 0000000000..5c46faa1e1 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/K.txt @@ -0,0 +1,8 @@ +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.len +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.is_empty +https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html#arithmetic-and-logical-binary-operators +https://doc.rust-lang.org/1.82.0/reference/types/numeric.html#integer-types +https://doc.rust-lang.org/1.82.0/reference/items/traits.html#unsafe-traits +https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html#unsafe-traits-unsafe-trait diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/M.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/M.txt new file mode 100644 index 0000000000..d74c62056e --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/M.txt @@ -0,0 +1,11 @@ +https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html +https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html +https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html +https://doc.rust-lang.org/1.80.0/reference/type-layout.html#primitive-data-layout +https://doc.rust-lang.org/1.80.0/std/primitive.u8.html#impl-Copy-for-u8 +https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html +https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html +https://doc.rust-lang.org/1.80.0/std/primitive.u32.html#impl-Copy-for-u32 +https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html +https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#impl-Copy-for-u32 +https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/Q.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/Q.txt new file mode 100644 index 0000000000..d467a88cf6 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/Q.txt @@ -0,0 +1 @@ +https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/R.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/R.txt new file mode 100644 index 0000000000..7515068ad3 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/R.txt @@ -0,0 +1,2 @@ +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut +https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#method.wrapping_add diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/S.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/S.txt new file mode 100644 index 0000000000..f13c981a34 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/S.txt @@ -0,0 +1,12 @@ +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.86.0/releases.html#version-1841-2025-01-30 +https://doc.rust-lang.org/1.86.0/releases.html#version-1851-2025-03-18 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/W.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/W.txt new file mode 100644 index 0000000000..655f04c58c --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/W.txt @@ -0,0 +1,2 @@ +https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety +https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/X.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/X.txt new file mode 100644 index 0000000000..25e171dfb3 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/X.txt @@ -0,0 +1,22 @@ +https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html#method.new_unchecked +https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html#comparison-operators +https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html +https://doc.rust-lang.org/1.85.1/std/macro.panic.html +https://doc.rust-lang.org/1.85.1/std/env/fn.var.html +https://doc.rust-lang.org/1.85.1/std/string/struct.String.html#method.as_str +https://doc.rust-lang.org/1.85.1/std/primitive.str.html#impl-ToOwned-for-str +https://doc.rust-lang.org/1.85.1/std/fmt/index.html#syntax +https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html +https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html +https://doc.rust-lang.org/1.85.1/reference/patterns.html#literal-patterns +https://doc.rust-lang.org/1.85.1/reference/patterns.html#or-patterns +https://doc.rust-lang.org/1.85.1/reference/patterns.html#wildcard-pattern +https://doc.rust-lang.org/1.85.1/std/macro.println.html +https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-cfg +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-check-cfg +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rerun-if-env-changed +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#life-cycle-of-a-build-script +https://doc.rust-lang.org/1.85.1/cargo/reference/features.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/authority-manifest.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/authority-manifest.tsv new file mode 100644 index 0000000000..bead2c56bb --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/authority-manifest.tsv @@ -0,0 +1,77 @@ +mode requested_url fetch_url final_url status content_type bytes sha256 fragment_found retrieved_utc +S https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +S https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +S https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html 200 text/html 213924 4e91aab56b50dbf245e6a1ea950ad74f4cb8971de958a7d4658d964a0a6350f5 true 2026-08-01T01:28:38.445793Z +S https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html 200 text/html 213924 4e91aab56b50dbf245e6a1ea950ad74f4cb8971de958a7d4658d964a0a6350f5 true 2026-08-01T01:28:38.445793Z +S https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +S https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +S https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html 200 text/html 214310 3e342d6b7b88a60eef09f8452d071040bf2e281e7d8228aaf727740be4015a15 true 2026-08-01T01:28:38.575213Z +S https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html 200 text/html 214310 3e342d6b7b88a60eef09f8452d071040bf2e281e7d8228aaf727740be4015a15 true 2026-08-01T01:28:38.575213Z +S https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +S https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +S https://doc.rust-lang.org/1.86.0/releases.html#version-1841-2025-01-30 https://doc.rust-lang.org/1.86.0/releases.html https://doc.rust-lang.org/1.86.0/releases.html 200 text/html 993212 5a25142ffcdc8a32d27623a01bde7e05f5dda96c172f87aa5d2b30e465c28998 true 2026-08-01T01:28:38.732157Z +S https://doc.rust-lang.org/1.86.0/releases.html#version-1851-2025-03-18 https://doc.rust-lang.org/1.86.0/releases.html https://doc.rust-lang.org/1.86.0/releases.html 200 text/html 993212 5a25142ffcdc8a32d27623a01bde7e05f5dda96c172f87aa5d2b30e465c28998 true 2026-08-01T01:28:38.732157Z +C https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +C https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +C https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_or https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +C https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +C https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +C https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_or https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +C https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +C https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +C https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_or https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +C https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html 200 text/html 36314 c231b94b17857c57b47336ebede1e3e81ee8d58db68e669a6b1da1026164ac3b true 2026-08-01T01:28:38.772524Z +C https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html 200 text/html 36314 c231b94b17857c57b47336ebede1e3e81ee8d58db68e669a6b1da1026164ac3b true 2026-08-01T01:28:38.772524Z +C https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html 200 text/html 6436 b92f2f9939bf888edd27d156eea4f5a48a98867752198001dc6b12253973fedd true 2026-08-01T01:28:38.811777Z +C https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:38.854337Z +C https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:38.854337Z +C https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html 200 text/html 6436 2a9462a947c44b60b1131aa5f1378b8ebbb47df234aa152232d935f5a95bc8bc true 2026-08-01T01:28:38.891018Z +C https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html 200 text/html 39866 f5ec1c4381d47fa28bbfe7b92f0c4383c0f8a30bc7121831fa4a4cd4a5543f4c true 2026-08-01T01:28:38.934329Z +C https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html 200 text/html 39866 f5ec1c4381d47fa28bbfe7b92f0c4383c0f8a30bc7121831fa4a4cd4a5543f4c true 2026-08-01T01:28:38.934329Z +C https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html 200 text/html 6503 0aa2780471d5ac4b0b5e746ea7aa01eb78fbabf9cd584a96d3088ec1afcc5883 true 2026-08-01T01:28:38.971684Z +X https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html#method.new_unchecked https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html 200 text/html 1396156 c1393af3fca7ba25fd448504d6f70fc01fea71efac9cc50052178899ccb877dc true 2026-08-01T01:28:39.066223Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html#comparison-operators https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html 200 text/html 58441 da30ee41874a27f0d5d991a9f53dda6d4b59e46e8f28ad06c0c714b941a9d3e0 true 2026-08-01T01:28:39.115780Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html 200 text/html 16334 66cb23b4db83562de51a0b3411d473b430c738aa934c44b75f30aa2dd02c8f05 true 2026-08-01T01:28:39.154617Z +X https://doc.rust-lang.org/1.85.1/std/macro.panic.html https://doc.rust-lang.org/1.85.1/std/macro.panic.html https://doc.rust-lang.org/1.85.1/std/macro.panic.html 200 text/html 10472 c7f3fe8e00889d1d9206fa39a373cfd060b993c5377b169979cc947e9efcecc0 true 2026-08-01T01:28:39.192926Z +X https://doc.rust-lang.org/1.85.1/std/env/fn.var.html https://doc.rust-lang.org/1.85.1/std/env/fn.var.html https://doc.rust-lang.org/1.85.1/std/env/fn.var.html 200 text/html 5859 4254c02ef260e2a7729980cc6f84796a5b3770ac58334a598fc0b745df2d805f true 2026-08-01T01:28:39.231253Z +X https://doc.rust-lang.org/1.85.1/std/string/struct.String.html#method.as_str https://doc.rust-lang.org/1.85.1/std/string/struct.String.html https://doc.rust-lang.org/1.85.1/std/string/struct.String.html 200 text/html 577036 d94dac44c1c9d360c7caf0806c0c41b4d2330913463d0be42eeb211416563006 true 2026-08-01T01:28:39.310266Z +X https://doc.rust-lang.org/1.85.1/std/primitive.str.html#impl-ToOwned-for-str https://doc.rust-lang.org/1.85.1/std/primitive.str.html https://doc.rust-lang.org/1.85.1/std/primitive.str.html 200 text/html 563520 f6569e4e89b98037dbba1106d5e1c7d0598d6592c48eaa6a42779e72430ca988 true 2026-08-01T01:28:39.395145Z +X https://doc.rust-lang.org/1.85.1/std/fmt/index.html#syntax https://doc.rust-lang.org/1.85.1/std/fmt/index.html https://doc.rust-lang.org/1.85.1/std/fmt/index.html 200 text/html 65822 3d3493d16ea89a7626d9f5036456924d617de6d50002b2c0ed4a55ba36be53a7 true 2026-08-01T01:28:39.444849Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html 200 text/html 23305 eca5d1df45aaac303bdaa0a7a0a7e93f2e48b38171325e963bed90d3dbaf31ba true 2026-08-01T01:28:39.485645Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html 200 text/html 17731 281bbc7dfe2c0f330e7229684b55747cd3392845fad3a5be536ec5a4918ed695 true 2026-08-01T01:28:39.560464Z +X https://doc.rust-lang.org/1.85.1/reference/patterns.html#literal-patterns https://doc.rust-lang.org/1.85.1/reference/patterns.html https://doc.rust-lang.org/1.85.1/reference/patterns.html 200 text/html 58174 4e3e6090c0a65f5d26e7ae807f4241f35b91e8fbd24a76ecb23a07bf4fd6d0cf true 2026-08-01T01:28:39.614415Z +X https://doc.rust-lang.org/1.85.1/reference/patterns.html#or-patterns https://doc.rust-lang.org/1.85.1/reference/patterns.html https://doc.rust-lang.org/1.85.1/reference/patterns.html 200 text/html 58174 4e3e6090c0a65f5d26e7ae807f4241f35b91e8fbd24a76ecb23a07bf4fd6d0cf true 2026-08-01T01:28:39.614415Z +X https://doc.rust-lang.org/1.85.1/reference/patterns.html#wildcard-pattern https://doc.rust-lang.org/1.85.1/reference/patterns.html https://doc.rust-lang.org/1.85.1/reference/patterns.html 200 text/html 58174 4e3e6090c0a65f5d26e7ae807f4241f35b91e8fbd24a76ecb23a07bf4fd6d0cf true 2026-08-01T01:28:39.614415Z +X https://doc.rust-lang.org/1.85.1/std/macro.println.html https://doc.rust-lang.org/1.85.1/std/macro.println.html https://doc.rust-lang.org/1.85.1/std/macro.println.html 200 text/html 6697 05b1c308b73cceceb09edd797b1cfb51b88039c26c9cd018aeda106fad2aa076 true 2026-08-01T01:28:39.653511Z +X https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:39.693555Z +X https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:39.693555Z +X https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html 200 text/html 6436 c9f3ba7f85fed765c6485ac0754891ee29b9f68efee68d60967a547264ffb748 true 2026-08-01T01:28:39.731921Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-cfg https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-check-cfg https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rerun-if-env-changed https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#life-cycle-of-a-build-script https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/features.html https://doc.rust-lang.org/1.85.1/cargo/reference/features.html https://doc.rust-lang.org/1.85.1/cargo/reference/features.html 200 text/html 36803 96b2337cd60180df5a8566f343e52938dfcaa369bf12e1e82723a2326f64cb25 true 2026-08-01T01:28:39.818858Z +Q https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html 200 text/html 11603 555597c0db28f65466dd734a9f57d4aaca8abe7f6e0e256b3f0d64a877529fd3 true 2026-08-01T01:28:39.858491Z +W https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html 200 text/html 11603 555597c0db28f65466dd734a9f57d4aaca8abe7f6e0e256b3f0d64a877529fd3 true 2026-08-01T01:28:39.858491Z +W https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html 200 text/html 37852 9f6deb0ebbdddd0362a3406ccb872c84108034c55c7d5b3e124d50b5d2cae9a9 true 2026-08-01T01:28:39.900179Z +M https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html 200 text/html 9404 f31ae6a889da541592c796e521b7abb18fa4b7a4dc25da32e839d076fe8aeba3 true 2026-08-01T01:28:39.944111Z +M https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html 200 text/html 9404 3c9e77c0d03f9669ba5342cff73a408360e0cc6bda1bc17a368234cf77f5ae4c true 2026-08-01T01:28:39.984139Z +M https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html 200 text/html 10246 c708643d37fa87df877526f34c6bc3021e0fd17c062e29b854da58b0b98843b4 true 2026-08-01T01:28:40.022195Z +M https://doc.rust-lang.org/1.80.0/reference/type-layout.html#primitive-data-layout https://doc.rust-lang.org/1.80.0/reference/type-layout.html https://doc.rust-lang.org/1.80.0/reference/type-layout.html 200 text/html 55218 f1e8382edb288ae23f8a2e654910addf3540f48f635bec80f0242f3c58b4b78c true 2026-08-01T01:28:40.070245Z +M https://doc.rust-lang.org/1.80.0/std/primitive.u8.html#impl-Copy-for-u8 https://doc.rust-lang.org/1.80.0/std/primitive.u8.html https://doc.rust-lang.org/1.80.0/std/primitive.u8.html 200 text/html 1121182 ee0cd4ff653be0fcb8144e42a9649a7320d7b617ef4301c0a86c187936072230 true 2026-08-01T01:28:40.161517Z +M https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html 200 text/html 85880 007591ab85643b155a8498f28b30a7fa11a392063c142abe8d7311b13b039d5b true 2026-08-01T01:28:40.212440Z +M https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html 200 text/html 11884 f783f80b1c0c29b052669635d24a767637a0b01dbc1c7c4f670154e59df3bbae true 2026-08-01T01:28:40.249944Z +M https://doc.rust-lang.org/1.80.0/std/primitive.u32.html#impl-Copy-for-u32 https://doc.rust-lang.org/1.80.0/std/primitive.u32.html https://doc.rust-lang.org/1.80.0/std/primitive.u32.html 200 text/html 1084587 ab99f0b7800c1ead37ea3857fbc8744f68d3ed1d0ddaf92fa49e395baf3bc146 true 2026-08-01T01:28:40.338022Z +M https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html 200 text/html 11904 ecb229ccdfc771ac09182b4abe0930e2b14013ad24c91340d7ad8e8350a3360a true 2026-08-01T01:28:40.377659Z +M https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#impl-Copy-for-u32 https://doc.rust-lang.org/1.82.0/std/primitive.u32.html https://doc.rust-lang.org/1.82.0/std/primitive.u32.html 200 text/html 1089970 15962c2e26e219d0df334635c8927ad7bb9007a38e237b529a61dce88aa4f4ef true 2026-08-01T01:28:40.469555Z +M https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html 200 text/html 86272 8f1114c00d2d8c08eea30caaa07257a7a018fd871a8d1e4598f55084be78a2e8 true 2026-08-01T01:28:40.520539Z +R https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +R https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#method.wrapping_add https://doc.rust-lang.org/1.82.0/std/primitive.u32.html https://doc.rust-lang.org/1.82.0/std/primitive.u32.html 200 text/html 1089970 15962c2e26e219d0df334635c8927ad7bb9007a38e237b529a61dce88aa4f4ef true 2026-08-01T01:28:40.469555Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.len https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.is_empty https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html#arithmetic-and-logical-binary-operators https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html 200 text/html 71217 9a08c53fb94c774ec5e4ddc069672f912f879716db7c17f649174afec3c72561 true 2026-08-01T01:28:40.659870Z +K https://doc.rust-lang.org/1.82.0/reference/types/numeric.html#integer-types https://doc.rust-lang.org/1.82.0/reference/types/numeric.html https://doc.rust-lang.org/1.82.0/reference/types/numeric.html 200 text/html 29770 7c21b867c4ba43f66984ece635049dedb96783d8de0490750cd2f1ee0fe44ead true 2026-08-01T01:28:40.703590Z +K https://doc.rust-lang.org/1.82.0/reference/items/traits.html#unsafe-traits https://doc.rust-lang.org/1.82.0/reference/items/traits.html https://doc.rust-lang.org/1.82.0/reference/items/traits.html 200 text/html 43949 8074f9255f3de0cc71edad5cbfbd22aeb8ac5427568c8478783d2aaca15d8363 true 2026-08-01T01:28:40.752604Z +K https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html#unsafe-traits-unsafe-trait https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html 200 text/html 33136 0de12b87f6a3967939624241cf67fc897de0eb93eca2b893c58dd1b734c4e0ea true 2026-08-01T01:28:40.797781Z diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/file-manifest.sha256 b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/file-manifest.sha256 new file mode 100644 index 0000000000..072b7e20e6 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/file-manifest.sha256 @@ -0,0 +1,47 @@ +a8ff68e702e9e4daa51a5560f39366f08e406df6eb8b1805484fa560a36cd0f8 fetch_authority.py +ad025876ef66f6e6578eea565088eaa99c24c0a603391d1e9f924858b3f63c48 freeze/allowlists/C.txt +b6a645f1b58c1ed0dab7480352ce18df11fe7a02468fd3df7fb6b403bc40d965 freeze/allowlists/K.txt +48b51034360f118b44292431a6776ee1eb24437abbd86a639e26fdd845cb3c70 freeze/allowlists/M.txt +3dc7862cdc2fdc1d44fd97235ad043602bd26ab3b428209b94fe9fb0602c14f8 freeze/allowlists/Q.txt +1cef79dd2eaef588d2343439d0b1a54b816adebf188a59fc6349f45055872ea4 freeze/allowlists/R.txt +c800f0c77ac630cbe60f1068cd172d32bc711d0a8ebe918c609996a8a3141977 freeze/allowlists/S.txt +c23fd5c190f565c18ee04ecf9918afd6463e434b573821b34022db16f594fa4e freeze/allowlists/W.txt +af068f9a92b8ae31e27cf9ee34075c54dd7f44906cc7e3ccb4b2369ad1e11b56 freeze/allowlists/X.txt +329634ba9421a9a65b2802b4ba5b07667fae194411904cbbfc7f002da84c0f97 freeze/authority-manifest.tsv +66522a17184a81a7832fc6a941bb947610c8971e29e28dbd7e64940542b198a2 freeze/manifest.json +982a13036918d561a6eb83d5effefe200ecdf81d6044b126c61c9017f1f91b2c freeze/oracle/controls.md +f6bddbbbd8edd032d257baf8c3b8d54731d1a3c0b284579dee9ec082887d5a30 freeze/oracle/domain.md +da2338aaed226d01c3a1a1399186dfc4b7c15b112321d65ddf883d405a56f961 freeze/oracle/verdict.md +76ed204e0bb03e546181bfef49f0d7cdf0c307c4d893e524e56820615cd8b0a7 freeze/plan.md +30df9df19f2dfa0056c0247f1d2e816bf0366d0f1d7eeacbea6755ed45f87117 freeze/policies/isolation.md +699fea44c59e993081c13fe9af7318e6a4b8bb4130b013b297f5256938b13405 freeze/policies/reruns.md +040a94fa045cc8baa4df28cddd1b7f18d4163561530351a73b1aa3578b86299d freeze/policies/tools.md +18e7b2bc357d82be6ca698d65a34bd73a53d321ce481e856a715b9b88bc47526 freeze/policies/word-count.md +1b67f22e8d221bdbc61d3ac700950b6fe140079a1c9d45736e1fe2cdb917abaf freeze/prompts/adjudicator.md +5c001fa4b647218d5c7a98468f66260b95d490fbdfc64b64217777b995df2782 freeze/prompts/report.md +ae7649ba7f57828ecff8f3c8c2b41987c65d67ff0cfb4142f7a17e08b7b55a5a freeze/prompts/scorer.md +39c4c657b39d2279dd3656e0f231906f3dfefdb853d24914616168b8c26c8bda freeze/randomization/commitments.json +75dbf874dbec04ee4738cfafa66d01ffba5510b81539662d8fb9596b43e3ae05 freeze/randomization/spec.md +3953c17ad001ee677061118952a5ac4fae44e21cc1ed1d64a0a0b2ae41c0049c freeze/rubrics/C.md +66ceb460ffb0893d9df8321707889c6f122fe0ccd9ec674fb31b07eb1ce5793e freeze/rubrics/K.md +d06d60ffd6b3478500caabb2ca88bb4b9327a7c63a3f0d913633ecb9a837d6e2 freeze/rubrics/M.md +154645b5d02c12a09d8162c6daca9d8843e81c9d4caa03165dc4acb5391d2a0a freeze/rubrics/Q.md +a3e9b8d28e8b9d4eafc5f9c4b553d6b862a623b2f786e1e5e806ea212317bc18 freeze/rubrics/R.md +d215ac1a4ed2e150f369b28aa12a4d77cf391c94ce47a98fccf132ad6a0c60f3 freeze/rubrics/S.md +b7ad63a8a816034852d1542e904eccce33c3e9ff60e6b4b2bb7f37f503ce8b82 freeze/rubrics/SCORER.md +c9aa4d310e2dc98d905caa3015c52c0cddbdcf95a1de65ea3a2f4fd78fef048c freeze/rubrics/W.md +ed88cb738258d69cf2e410f33fc2cabe7a209fecf36440b23707f3b9ebaeda36 freeze/rubrics/X.md +58d36e6a54002a29fe7f45c4f65d966a307212e6d082be75d34772fc5b128209 freeze/schemas/adjudication.schema.json +f606e610ef90dde0a7b5c425390ec44b6d4ce7059e95b2028feee9f0a8578381 freeze/schemas/disagreements.schema.json +d59caebb2439606adb0c64d1545732d3feaef93e5d10afa7eeba92d9a2623f3b freeze/schemas/event.schema.json +a54869ce1a3a4ac0cfa23ca63ec11a507406a5ae42ab09ad7a73cb5b444c729d freeze/schemas/final.schema.json +5b0563377e75d962d68ecace1225a73985fbb33ab422051377a0926cc7bb88d3 freeze/schemas/score.schema.json +1ebdd275e5b46a500de5028ffec6d30e48196b970cde80d07c72b73927038eba prepare.py +8d7ac565b05790805b4d79646c1ebb8dfd965f348ed05bf7c089276cc343aa0f protocol.py +8e739c0fcd05e19910d4dc986eebed31bd2629fc8d608ec6f86f15c38a6dfe75 sealed/blind-map.tsv +abd22fffee16a0629e9b721f2226c068a187949e05cba4711ee7a6bcbb5a02cb sealed/condition-map.tsv +03809f6698d6289e15f25434dbc931aed3619387d5a6ea586c0b0fc6d0f969ff sealed/launch-schedule.tsv +dfb7db6e6e6b35eb72034b9de3eece05e213e3e9ef2f285bca89d7352a5ea29b sealed/presentation-orders.tsv +7fec2919ec0b489c1c35a07b20123ffcde18f3a66958f87d87de4d7646a71ffd sealed/scoring-schedule.tsv +364231790dd4797101619bfd4bc1b538df0a194c7a066dddeba9fea2be7af368 sealed/seeds.json +b7809a7c771b203e0652b397ba9b57ef974424e474f87d5c740544715fdf7d84 sealed/target-map.tsv diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/manifest.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/manifest.json new file mode 100644 index 0000000000..6069c1f579 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/manifest.json @@ -0,0 +1,101 @@ +{ + "schema_version": 1, + "run_id": "2026-07-31-v3-targeted", + "status": "FROZEN_WHEN_LOCKED", + "prepared_utc": "2026-08-01T01:27:31Z", + "source_control": { + "repository_parent_commit": "47d01d20f04e4c5d4498a2f3ba3a098d370fd79c", + "branch": "unsafe-code-skill", + "note": "The freeze commit is created only after LOCK.json; every precommit freeze byte is independently bound by file-manifest.sha256." + }, + "objective": { + "primary": "Absolute confirmation of the frozen V3 capabilities represented by all eight mode rubrics.", + "diagnostic_comparator": "The coherent frozen V2 package; it is not a causal ablation.", + "release_claim": "A pass permits a later broad release gate but is not itself release readiness." + }, + "design": { + "modes": ["S", "C", "X", "Q", "W", "M", "R", "K"], + "conditions": 2, + "replicates_per_cell": 5, + "expected_reports": 80, + "balanced_waves": 5, + "maximum_concurrent_report_agents": 3, + "dual_blind_scorers_per_mode": 2, + "freshness": "fork_turns=none for every report, scorer, and adjudicator agent" + }, + "inference": { + "model": "gpt-5.6-sol", + "reasoning_effort": "ultra", + "service_tier": "priority", + "sampling_seed": null, + "hosted_model_build": null, + "limitation": "The orchestration API exposes neither a sampling seed nor an exact hosted model-build identifier." + }, + "packages": { + "v3": { + "role": "confirmatory candidate", + "tree_sha256": "668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf", + "skill_sha256": "0e23f7747cc63014bade7543efaf745e7e9a7e5d6dee2a48c602ef7a3eba091e" + }, + "v2": { + "role": "diagnostic comparator", + "tree_sha256": "40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897", + "skill_sha256": "a0a75ef8a14497aa78b50b459981097ee99605c57fec95c637cf59aaa20fe766" + } + }, + "targets": { + "S": "28ecc523e15b914a187814ab2752c0d85996948a552c62f970d8484bc6ed467a", + "C": "065c3cfc032af93e7576e17e49322826c4a379a707870175b16c2663d1e8e4e0", + "X": "25b4efef689601f3b5983bf6b914bd367153d0b21a6f1f4d40de50c5d412afa7", + "Q": "c0a4c43373a159cb38d08724af8b02187b249ab6a73f7e1b10b2276f38b0cb5a", + "W": "b27b95fbc9ffa9d335bb6b4614a9f227a5798122fca79b42cf53c9b106a6aff6", + "M": "b269cf068196d1c06b87be6bcded494827e7ac8a2cb6debf1eb0f0f5d0388479", + "R": "6d1a41909b012484d71f94d194071a7ebbb6773b7596db88127ca2a195b70ffc", + "K": "ca272e524b36a892e25f6169631a184ff764026451eff2f6ac4ab8e9d5e87ea2" + }, + "authority": { + "policy": "Only direct opens of exact URL-only per-mode allowlists are permitted; search and link traversal are prohibited.", + "manifest": "authority-manifest.tsv", + "allowlist_entries": 76, + "distinct_retrieved_pages": 46, + "retrieval_started_utc": "2026-08-01T01:28:38.381052Z", + "retrieval_finished_utc": "2026-08-01T01:28:40.797781Z", + "live_page_rule": "Immediately before each collection wave, fetch_authority.py must reproduce every frozen byte count, SHA-256, final URL, MIME type, and fragment. Earlier waves must be complete and no cell in the new wave may be prepared; drift blocks the run.", + "limitation": "Agents open live versioned pages rather than an evaluator-hosted mirror; network allowlisting is procedural on the shared host." + }, + "integrity": { + "tree_digest": "GNU tar 1.x: tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner -C TREE -cf - . | SHA-256; symlinks are rejected.", + "freeze_root": "SHA-256 of the byte-exact sorted file-manifest.sha256", + "randomization": "Five distinct nonzero 256-bit seeds and all derived maps are sealed and lock-covered; seed generation provenance is not externally attested.", + "events": "events.jsonl uses sequential canonical JSON records chained by SHA-256 of the preceding line; one run-wide operation lock serializes all protocol state transitions.", + "isolation": "Runtime paths and packets are neutral and inputs are read-only, but filesystem and network isolation are procedural rather than hardened." + }, + "canonical_inputs": { + "plan": "plan.md", + "oracles": ["oracle/domain.md", "oracle/verdict.md", "oracle/controls.md"], + "rubrics": "rubrics/", + "allowlists": "allowlists/", + "prompts": "prompts/", + "policies": "policies/", + "schemas": "schemas/", + "randomization": "randomization/", + "sealed_maps": "../sealed/", + "generator": "../prepare.py", + "authority_verifier": "../fetch_authority.py", + "protocol_and_aggregator": "../protocol.py" + }, + "primary_gates": { + "every_v3_atom": "PASS in 5 of 5 reports", + "v3_hard_errors": 0, + "v3_proposal_laundering": 0, + "v3_tcb_or_authority_defects": 0, + "v3_terminal_semantic_noncompletions": 0, + "v3_source_scope_or_budget_defects": 0 + }, + "known_nonclaims": [ + "No causal effect estimate follows from the coherent V3-versus-V2 comparison.", + "No cryptographic claim of condition blinding or packet-only access is made.", + "No target was built, tested, executed, or macro-expanded during fixture or freeze validation.", + "No report had been collected when this manifest was prepared." + ] +} diff --git a/evals/unsafe-rust/v3-targeted-drafts/controls.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/controls.md similarity index 85% rename from evals/unsafe-rust/v3-targeted-drafts/controls.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/controls.md index 2606f31bc9..f534317798 100644 --- a/evals/unsafe-rust/v3-targeted-drafts/controls.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/controls.md @@ -1,7 +1,6 @@ -# V3 Targeted Oracle Draft — R and K +# V3 Targeted Oracle — R and K -> Evaluator-only draft. Do not expose to report agents. Freeze only after two -> independent source and authority reviews. +> Evaluator-only material. Do not expose to report agents. ## R — abstraction redesign @@ -49,10 +48,13 @@ need. ### Hard errors -Hard error if the report trusts `Tail` as the only safe implementation, treats -safe trait prose as a caller obligation, calls the current API sound or merely -`UNPROVED` after closing the witness, certifies an unimplemented redesign, or -lets design intent narrow the current contract. +Hard error if the report: + +- trusts `Tail` as the only safe implementation; +- treats safe trait prose as a caller obligation; +- calls the current API sound or merely `UNPROVED` after closing the witness; +- certifies an unimplemented redesign; or +- lets design intent narrow the current contract. ### Authority @@ -121,16 +123,23 @@ each contract; do not infer the second disposition from the first. ### Hard errors -Hard error if the report trusts arbitrary safe `Position` implementations, -misses either safe surface while claiming complete coverage, calls either -surface sound, rejects the correct `last` implementation because its comment is -bad, explicitly calls the deficient comment adequate, relies on that comment as -the proof, claims complete requested coverage while omitting the reconstruction, -weakens the published trait contract from visible use alone, treats sealing -alone as satisfying the stated functional repair, claims a callable -caller-position API is repaired without locally coupling the selected position -to the particular slice, or certifies an unimplemented repair. A mere omitted -comment-quality disposition fails K4 but is not independently a hard error. +Hard error if the report: + +- trusts arbitrary safe `Position` implementations; +- misses either safe surface while claiming complete coverage; +- calls either safe surface sound; +- rejects the correct `last` implementation because its comment is bad; +- explicitly calls the deficient comment adequate; +- relies on that comment as the proof; +- claims complete requested coverage while omitting the reconstruction; +- weakens the published trait contract from visible use alone; +- treats sealing alone as satisfying the stated functional repair; +- claims a callable caller-position API is repaired without locally coupling + the selected position to the particular slice; or +- certifies an unimplemented repair. + +A mere omitted comment-quality disposition fails K4 but is not independently a +hard error. ### Authority diff --git a/evals/unsafe-rust/v3-targeted-drafts/domain.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/domain.md similarity index 99% rename from evals/unsafe-rust/v3-targeted-drafts/domain.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/domain.md index 2b93b8b8e7..174024dd31 100644 --- a/evals/unsafe-rust/v3-targeted-drafts/domain.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/domain.md @@ -1,4 +1,4 @@ -# V3 Targeted Domain Evaluator Draft +# V3 Targeted Domain Oracle > **Evaluator-only material.** Never expose this file, its mode labels, atoms, > intended predicates, hard-error rules, or authority list to an evaluated @@ -462,13 +462,13 @@ For effective rejection, verify that `cfg(all(...))` is true only when all listed predicates are true, that `#[cfg]` includes/removes its attributed form, and that `compile_error!` causes compilation to fail: -- `https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html` -- `https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html` -- `https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html` @@ -509,7 +509,7 @@ Exact pages: - `https://doc.rust-lang.org/1.85.1/reference/patterns.html#or-patterns` - `https://doc.rust-lang.org/1.85.1/reference/patterns.html#wildcard-pattern` - `https://doc.rust-lang.org/1.85.1/std/macro.println.html` -- `https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html` @@ -525,4 +525,4 @@ review that entry, but do not enlarge it: - `https://doc.rust-lang.org/1.85.1/cargo/reference/features.html` No release blog, CI outcome, execution result, Miri result, prior report, or -this evaluator draft is an authoritative Rust semantic premise. +this evaluator oracle is an authoritative Rust semantic premise. diff --git a/evals/unsafe-rust/v3-targeted-drafts/verdict.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/verdict.md similarity index 96% rename from evals/unsafe-rust/v3-targeted-drafts/verdict.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/verdict.md index 462a52b776..1f3f340224 100644 --- a/evals/unsafe-rust/v3-targeted-drafts/verdict.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/verdict.md @@ -1,12 +1,10 @@ -# V3 Targeted Evaluator Draft — Verdict and Release Modes +# V3 Targeted Oracle — Verdict and Release Modes > **Evaluator-only material.** Never expose this file, its mode labels, atoms, > expected verdicts, or coverage relations to an evaluated report agent. > -> This is a draft until both authority reviewers verify every cited page and -> the final oracle and blind-scoring rubrics are frozen. Score propositions, -> not keywords or report layout. Equivalent explicit derivations pass. Do not -> infer a missing material link from vague shorthand. +> Score propositions, not keywords or report layout. Equivalent explicit +> derivations pass. Do not infer a missing material link from vague shorthand. ## Common scoring rules @@ -23,7 +21,7 @@ mode-specific rule below or the frozen global rules say so. ## Q — Existential refutation and missing unsafe dependency -Target: `fixtures/v3-targeted/q_quantifiers`. +Target: supplied Q snapshot. Let `C_Q` be Rust 1.80.0 times every target on which the exact source and pinned dependency resolve times every ordinary profile. @@ -85,7 +83,7 @@ Any of these is a hard error: - treating the unsafe dependency's documented caller precondition as though violating its prose were itself an executed UB event. -### Q authority to verify before freezing +### Q checked authority - `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` — exact proposition consumed: reaching @@ -96,7 +94,7 @@ No Rust authority is asserted for the absent dependency implementation. ## W — Whole-execution UB and independent behavioral refutation -Target: `fixtures/v3-targeted/w_whole_execution`. +Target: supplied W snapshot. Let `C_W` be Rust 1.80.0 times every target on which this exact source and the used Rust 1.80.0 standard-library item exist times every ordinary profile, and @@ -146,7 +144,7 @@ Any of these is a hard error: - falsely treating the independent input-one refutation as UB-containing, or claiming complete behavioral coverage while omitting it. -### W authorities to verify before freezing +### W checked authorities - `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` — reaching the function is undefined behavior. @@ -159,7 +157,7 @@ Any of these is a hard error: ## M — Multi-release affirmative certificates -Target: `fixtures/v3-targeted/m_multirelease`. +Target: supplied M snapshot. Let `C_M` be every supported target times every ordinary profile. For a function `f`, let `Calls_f` be all calls satisfying its documented safety @@ -288,7 +286,7 @@ Failure to establish a required positive regional result is an atom failure. It is not by itself a hard error unless the report also makes one of the false affirmative or scope-changing claims above. -### M authorities to verify before freezing +### M checked authorities Open each exact page and confirm the named description and Safety propositions, including all qualifications relevant to its exact release case: diff --git a/evals/unsafe-rust/v3-targeted-plan.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/plan.md similarity index 81% rename from evals/unsafe-rust/v3-targeted-plan.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/plan.md index 19972dcab9..00d26c42c0 100644 --- a/evals/unsafe-rust/v3-targeted-plan.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/plan.md @@ -2,9 +2,10 @@ > **Evaluator-only material. Do not expose this file to evaluated agents.** > -> **Preregistration status:** DRAFT. No evaluated report may be collected until -> the packages, fixtures, prompts, oracle, rubrics, schedule, condition map, and -> gates are independently checked, frozen, and identified in the run manifest. +> **Preregistration status:** FROZEN. This status is effective only with a valid +> `LOCK.json` whose root matches `file-manifest.sha256`. No evaluated report may +> be collected before that lock exists and passes `protocol.py verify-static +> --locked`. ## Purpose @@ -45,7 +46,8 @@ Use eight focused modes, two conditions, and five fresh replicates per cell: Each evaluated agent receives one opaque package, one opaque target, and one empty output directory. Reports are randomized across conditions and modes. -Condition identity is revealed only after reports have been preserved, hashed, +The condition map is generated and sealed before collection. Condition identity +is revealed to the analysis only after reports have been preserved, hashed, blind-scored twice, and adjudicated. Five replicates are an engineering reliability minimum, not a population-level @@ -136,7 +138,8 @@ The targeted confirmation passes only if all of the following hold: 3. V3 has zero proposal laundering. 4. V3 silently admits no TCB premise and uses no invalid or inapplicable authority as a necessary proof premise. -5. Every V3 report respects the frozen source-only scope and word budget. +5. V3 has zero terminal semantic noncompletions. +6. Every V3 report respects the frozen source-only scope and word budget. Failure of any primary gate fails the run. Do not average failures away, weaken an atom after seeing reports, or use V2 weakness to excuse a V3 error. @@ -191,7 +194,19 @@ Before collection, freeze and hash: - one per-mode URL allowlist containing only exact official documentation URL identities, or a byte-identified mirror manifest; - the randomized schedule and condition map; -- output schema, word budgets, tool policy, and rerun policy. +- report-validation and aggregation programs, word budgets, tool policy, and + rerun policy. + +Every report, scorer, and adjudicator agent uses `gpt-5.6-sol` with reasoning +effort `ultra`, `fork_turns="none"`, and no helper agents. The orchestration API +does not expose a sampling seed or exact hosted model-build identifier; this is +an acknowledged reproducibility limit. Collection follows the sealed schedule +with at most three report agents active and a complete balanced-wave barrier. +Immediately before preparing each wave, the authority verifier must reproduce +the frozen official-document bytes. That wave may not be verified early: every +earlier wave must already be complete and no cell in the new wave may yet be +prepared. Protocol state transitions are serialized by one run-wide operation +lock. Each report agent must: @@ -201,20 +216,34 @@ Each report agent must: permitted official Rust/std pages (or frozen mirror bytes), and empty output directory; - avoid building, testing, executing, or macro-expanding the target; -- write one `report.md` and return the same report; +- write one canonical `report.md`; the final chat response is preserved only as + operational metadata and is never scored; - stay within 1,800 words, except that X receives a 2,400-word cap and K a 2,200-word cap, identical across conditions. Only genuine infrastructure failures may be rerun. Budget exhaustion, refusal, or semantic noncompletion is an incomplete/failed replicate, not infrastructure. Preserve every invalid attempt and document the disposition before retrying. +A terminal report noncompletion is blind-scored as produced; when no usable +`report.md` exists it receives an evaluator-marked canonical placeholder so +missing propositions fail without inventing agent work, and it independently +fails the zero-semantic-noncompletion gate. A non-rerunnable +invalid scorer or adjudicator output makes the run `INVALID`; no replacement +judgment is fabricated. + +The same attempt lifecycle applies to scorers and adjudicators. An API failure +before any agent identity exists is recorded but is not an attempt. A genuine +infrastructure failure after start preserves that numbered attempt and permits +exactly the next fresh attempt. Any non-infrastructure invalid evaluator output +is terminal and makes the run `INVALID`. ## Blind scoring and adjudication After collection and before unblinding: 1. Preserve and hash every raw report. -2. Assign random anonymous labels independently within each mode. +2. Materialize the pre-frozen anonymous-label map independently within each + mode. 3. Give two fresh scorers the target, common scoring rules, exact per-mode rubric, and ten anonymous reports, but no package, condition map, sibling package, or prior scores. @@ -223,6 +252,11 @@ After collection and before unblinding: 6. Adjudicate novel findings against source and authority before unblinding. 7. Preserve raw scores, adjudications, ledgers, and all integrity checks. +Blind scorers decide only scope defects visible in report content. The runner +independently counts words and records every known operational path/tool/source +deviation. Aggregation ORs those three sources into the source-scope/budget +gate; no scorer is asked to infer unavailable execution telemetry. + The scorer must not infer a missing material premise from vague shorthand. The rubric must state in advance which compact formulations count, especially where one conceptual defect admits multiple independently scored witnesses. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/isolation.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/isolation.md new file mode 100644 index 0000000000..ec4a61c77a --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/isolation.md @@ -0,0 +1,44 @@ +# Frozen Isolation and Packet Policy + +Each report cell receives a unique neutral runtime root with entries named only +`package`, `target`, `allowlist.txt`, and `output`. Inputs are copied +byte-for-byte, made read-only where the host permits, and verified against +frozen identities before launch and after the attempt. `output` begins empty. + +The shared collaboration host does not provide a hardened per-agent mount or +network allowlist. Isolation therefore remains procedural. Runtime inputs must +not contain a repository, `.git`, evaluator files, mode/condition names, or +cross-cell paths. This limitation is disclosed in every result and prevents a +claim of cryptographically enforced blinding. + +Before scorer launch, preserve and hash each canonical `report.md` and copy it +into a fresh mode packet under its frozen anonymous label. Chat-return text is +operational metadata and is not an alternative report channel. Prefer neutral +paths from collection so no report redaction is needed. Any required +normalization must use a frozen transformer and preserve the exact original, +transformed hash, and diff. + +Scorer packets contain only the target, allowlist, common rules, one mode +rubric, ten anonymous reports, score schema, and packet-local hashes. All packet +filesystem timestamps are normalized to the Unix epoch. Packets contain no +package, condition, run ID, collection order, agent identity, prior score, or +sibling mode. Adjudicator packets contain only the disputed reports/cells and +rationales, not unrelated agreed decisions. + +The append-only event ledger externally pins the canonical collection index +and every complete scorer and adjudicator packet byte tree. Packet verification +checks that external digest as well as the packet-local manifest before every +use. One run-wide operation lock serializes all state checks, artifact writes, +and event transitions. A canonical report, score, or adjudication and its +attempt record derive from the same single capture of the agent's output bytes; +the protocol does not reread a live output path to create either copy. + +Failure preservation remains possible when an evaluated agent changes a +runtime input despite the procedural restrictions. The protocol binds the +expected neutral path, records expected and safely observed input identities +without following symlinks, and snapshots the entire neutral runtime on an +input or inventory verification failure. The snapshot preserves every regular +byte and records every directory, symlink, and special entry without following +symlinks. It also snapshots the central setup or source packet whose identity +was checked. Report input drift becomes a terminal scope failure; evaluator +packet drift makes the run `INVALID`. It never authorizes a retry. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/reruns.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/reruns.md new file mode 100644 index 0000000000..8376a064f5 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/reruns.md @@ -0,0 +1,43 @@ +# Frozen Attempts, Reminders, and Rerun Policy + +Every attempt is immutable and receives its own directory. Preserve every +regular filesystem output byte, agent identity, timestamps, hashes, known +tool/path deviations, API completion/error state, and disposition before +considering another attempt. The canonical file is the sole evaluated channel; +chat-return prose need not be duplicated byte-for-byte. + +Only an externally established infrastructure failure may authorize a fresh +agent attempt. Examples are a service error before semantic work, a tool crash, +or a write failure caused by unavailable infrastructure. Refusal, word-budget +exhaustion, timeout after substantive work, invalid reasoning, missing atoms, +or other semantic noncompletion is a failed replicate and may not be rerun. + +An API failure that produces no agent identity is not an evaluated attempt; +record it separately as `API_NO_AGENT_START`, then launch the same next attempt +number. This applies to report, scorer, and adjudicator launches. For any +started attempt, use only these infrastructure disposition codes: +`SERVICE_ERROR_BEFORE_OUTPUT`, `ORCHESTRATOR_TOOL_FAILURE`, and +`FILESYSTEM_FAILURE`. Record the observed evidence and preserve every partial +artifact. A condition not fitting one of those codes is not rerunnable. + +A started scorer or adjudicator infrastructure failure likewise preserves its +own attempt directory and authorizes exactly the next numbered fresh attempt. +A schema-invalid, semantically incomplete, or otherwise non-infrastructure +scorer/adjudicator output is non-rerunnable and makes the evaluation `INVALID`. + +A terminal report-agent noncompletion is represented by its usable `report.md` +when one exists or by an evaluator-marked placeholder otherwise, is blind-scored +under the ordinary rubric, and is never rerun. Never fabricate a replacement +report, score, or adjudication. An empty or whitespace-only `report.md` is +semantic noncompletion, never a complete report. + +Preserved evaluator-attempt and invalid-output directories are inventoried in +both directions against unique ledger events. Any orphan directory, missing +directory, duplicate event, invalid hierarchy, `INVALID.json` marker, or +terminal-invalid ledger event fails closed. Once either terminal-invalid signal +exists, no later evaluation operation is permitted; a complete invalid state +must bind the marker, attestation, return event, and invalidation event exactly. + +One exact neutral reminder from `prompts/report.md` is permitted at 180 seconds. +No second reminder or substantive steering is permitted. Scorers and +adjudicators receive no reminders. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/tools.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/tools.md new file mode 100644 index 0000000000..5c1c8749ad --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/tools.md @@ -0,0 +1,16 @@ +# Frozen Tool and Source Policy + +Report agents may read only their per-cell package, target, URL-only allowlist, +allowed exact pages, and initially empty output directory. They may write only +`output/report.md` with `apply_patch`. Direct URL opens are permitted; web +search, link traversal, and every non-allowlisted page are prohibited. + +Targets must not be modified, built, tested, executed, or macro-expanded by +execution. Report agents may not spawn helper agents. They must not inspect +sibling/enclosing paths, repository or Git data, evaluator material, schedules, +maps, other cells, or prior reports. + +Scorers and adjudicators have analogous packet-only read scope and one-file +output scope. Tool and path restrictions are procedural on the shared host; +they are not a hardened sandbox. Every known deviation is preserved in the +append-only event ledger and fails the scope gate when material. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/word-count.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/word-count.md new file mode 100644 index 0000000000..c6b9eadd8c --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/word-count.md @@ -0,0 +1,22 @@ +# Frozen Word-Count Policy + +Decode `report.md` as UTF-8, split it on every Unicode whitespace run using +Python 3 `str.split()`, and count the resulting nonempty fields. Invalid UTF-8 +is an invalid attempt. + +The caps are: + +| Mode | Maximum words | +|---|---:| +| S | 1,800 | +| C | 1,800 | +| X | 2,400 | +| Q | 1,800 | +| W | 1,800 | +| M | 1,800 | +| R | 1,800 | +| K | 2,200 | + +The same cap applies to both conditions. A report above its cap remains +preserved but fails the source-scope/budget gate; it is not an infrastructure +failure and may not be rerun. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/adjudicator.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/adjudicator.md new file mode 100644 index 0000000000..52f90928e1 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/adjudicator.md @@ -0,0 +1,23 @@ +# Frozen Blind-Adjudicator Prompt + +```text +Act as a fresh blind adjudicator. Read every file under [PACKET], including the +target, URL allowlist, SCORER.md, RUBRIC.md, adjudication schema, disputed +anonymous reports, DISAGREEMENTS.json, and the two rationales for each disputed +cell. Follow the frozen instructions exactly. + +Resolve only the listed disagreements. Preserve every agreed decision. Decide +from the report's actual evidence: external authority may verify a premise the +report invokes, but may not add a material premise or derivation the report +omitted. Do not decide by majority, style, report similarity, or speculation +about a generating condition. + +Inspect only [PACKET] and exact URLs in its allowlist. Do not inspect siblings, +the enclosing repository, packages, condition maps, other scores, or other +adjudications. Do not modify the packet and do not spawn helper agents. + +Write exactly one UTF-8 JSON file, [OUTPUT]/adjudication.json, conforming to the +supplied schema. Create no other output file. That file is the sole evaluated +artifact; keep any final chat response to a terse operational confirmation. +Give compact decisive evidence and record genuine rubric ambiguity. +``` diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/report.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/report.md new file mode 100644 index 0000000000..2669f5bbba --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/report.md @@ -0,0 +1,36 @@ +# Frozen Evaluated-Agent Prompt + +Each report agent receives this text byte-for-byte except for resolving the +five bracketed placeholders. `[WORD_LIMIT]` is mode-specific but identical +between conditions within a mode. + +```text +Act as a fresh source-review agent. Read the complete unsafe Rust skill package +rooted at [PACKAGE]/SKILL.md and every reference it directs you to for this +task, then follow it. Read REQUEST.md and every file in [TARGET], and complete +exactly the requested review or design work. + +Your only permitted local inputs are [PACKAGE], [TARGET], [URL_ALLOWLIST], and +the empty [OUTPUT] directory. The allowlist contains the only external pages +you may inspect. Open those exact URLs directly if needed; do not search the +web or follow links to another page. Do not inspect a sibling or enclosing +directory, repository or version-control material, evaluator files, another +package or target, prior reports, schedules, or condition information. Treat +paths and opaque labels as non-semantic. + +Do not modify, build, test, execute, or macro-expand the target. Do not spawn +helper agents. Write exactly one UTF-8 file, [OUTPUT]/report.md, using +apply_patch, and create no other output file. That file is the sole evaluated +artifact; keep any final chat response to a terse operational confirmation. +Keep the report at or below [WORD_LIMIT] words, counting the nonempty fields +produced by splitting Unicode text on whitespace. This is a focused source +review; provide the complete proof material compactly. +``` + +No substantive steering is permitted. If the agent is still running 180 +seconds after launch, exactly one reminder may be sent: + +```text +Complete now within the frozen word limit using only material already +inspected; do not widen scope. +``` diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/scorer.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/scorer.md new file mode 100644 index 0000000000..bc42474f76 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/scorer.md @@ -0,0 +1,28 @@ +# Frozen Blind-Scorer Prompt + +Each scorer receives this text byte-for-byte except for resolved `[PACKET]` and +`[OUTPUT]` paths and its opaque `[SCORER_ID]`. + +```text +Act as a fresh blind evaluator. Read every file under [PACKET], including the +target, URL allowlist, SCORER.md, RUBRIC.md, score schema, and anonymous reports +A through J. Follow the frozen scoring instructions exactly and score every +report independently. Inspect reports in the `presentation_order` recorded in +PACKET.json; do not reorder them by similarity or suspected condition. + +Inspect only [PACKET] and the exact URLs in its allowlist. Open an allowed URL +directly if needed; do not search, follow links, or inspect any sibling or +enclosing directory, repository material, package, condition map, prior score, +or another scorer's output. Do not identify, cluster, or speculate about report +conditions. Do not modify the packet and do not spawn helper agents. + +Write exactly one UTF-8 JSON file, [OUTPUT]/score.json, conforming to the +supplied schema and using scorer_id [SCORER_ID]. Create no other output file. +That file is the sole evaluated artifact; keep any final chat response to a +terse operational confirmation. Score every atom and every independent defect +flag, cite compact report evidence for each decision, and record genuine rubric +ambiguity. Keep all evidence concise. +``` + +No reminder is permitted. An invalid or incomplete scorer output is preserved +and does not silently replace an independent score. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/commitments.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/commitments.json new file mode 100644 index 0000000000..8c0478a3e0 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/commitments.json @@ -0,0 +1,11 @@ +{ + "algorithm": "sha256(tag_utf8 || NUL || seed_bytes)", + "commitments": { + "blind": "f5c2a105dcd39c9a18b18e24a8b2393e9393a2d8cc9e2edd5bae3fa80fd32ed5", + "condition": "eb32738dca411d321e0bdd76b54a7ff82eedd0044d1bf1ce00de8bec201d190d", + "presentation": "6ee4618cc2757cd4a1fbfbcac0477247c3b346e73574610c62345ae07b9b1aa2", + "schedule": "23b61a3ae86f32f6262c96b46e6b5f8002d2a018c742ab7c371a58fb0c6ffe16", + "scorer": "395f3040683208aaf77eef98536646efdb83d9504bb0db00c3520b202ca587de" + }, + "schema_version": 1 +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/spec.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/spec.md new file mode 100644 index 0000000000..506a860055 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/spec.md @@ -0,0 +1,41 @@ +# Frozen Randomization Specification + +`prepare.py` is the executable specification. It uses five independently +generated 256-bit seeds, stored under evaluator-only `sealed/seeds.json`, and +domain-separated SHA-256 hash sorting. No language PRNG or iteration-order +behavior determines an order. + +For tag `T`, seed `S`, and canonical UTF-8 value `V`, the sort key is: + +```text +SHA256(T || NUL || bytes_from_hex(S) || NUL || V) +``` + +The canonical tuple breaks the cryptographically negligible event of a key +collision. + +The procedure is: + +1. Hash-sort real condition roles and assign opaque labels `c0` and `c1`. +2. Hash-sort mode names and assign opaque target labels `m0` through `m7`. +3. Treat each replicate as a balanced wave containing all eight modes and both + conditions. Hash-sort the five waves, then the sixteen cells within each + wave. Assign sequential operational run IDs only after sorting. +4. Derive each neutral 128-bit runtime cell ID from a separately tagged hash of + its canonical `(mode, role, replicate)` tuple. Assert all 80 are unique. +5. Independently hash-sort each mode's ten run IDs and assign blind labels A–J. +6. Independently hash-sort A–J for each scorer's presentation order. +7. Independently hash-sort the sixteen scorer claims for the scoring launch + order. + +The sealed maps are frozen before collection but never enter a report-agent or +blind-scorer packet. Because the shared repository and host are not hardened +against deliberate out-of-scope reads, secrecy is procedural. Commitments are +recorded in `commitments.json`; the seed and generated-map bytes are covered by +the freeze lock. + +Collection processes one complete balanced wave before the next. Within a +wave, starts follow `launch-schedule.tsv`; up to three report agents may be +active concurrently. A later service limitation may reduce concurrency but may +not reorder starts or cross the wave barrier. Every actual start/completion is +recorded in the append-only event ledger. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/C.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/C.md new file mode 100644 index 0000000000..cb18a8c259 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/C.md @@ -0,0 +1,128 @@ +# Mode C Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## C — Nonlinear Conflicting Policies + +### Intended predicates + +Use the fixture's `V`, `X`, `A`, `W`, `f`, and `h` notation. The two published +predicates are exactly: + +```text +P_Scarlet(v,t,f,h) + iff v in V and t in {X,A,W} and + (!f + or (f and t = X and (!h or v >= 1.85.0)) + or (f and t = A and h)). + +P_Indigo(v,t,f,h) + iff v in V and t in {X,A,W} and + (!f + or (f and t = X and (h or v >= 1.86.0)) + or (f and t = A and !h and v >= 1.85.0)). +``` + +Neither predicate contains the other. Representative disputed cases include: + +- Scarlet-only: `(1.84.0, X, turbo, !hardened)` and + `(1.84.0, A, turbo, hardened)`; +- Indigo-only: `(1.84.0, X, turbo, hardened)` and + `(1.85.0, A, turbo, !hardened)`. + +There is no authorized exact project predicate. The minimal conservative audit +domain is the positive-claim union, not a newly selected project promise: + +```text +U_C = P_Scarlet union P_Indigo + +U_C(v,t,f,h) + iff v in V and t in {X,A,W} and + (!f + or (f and t = X) + or (f and t = A and (h or v >= 1.85.0))). +``` + +Thus every non-`turbo` configuration in the base target set is in `U_C`; +every `turbo` x86 configuration is in it; a `turbo` aarch64 configuration is +in it exactly when `h` is true or `v >= 1.85.0`; and no `turbo` wasm32 +configuration is in it. + +For the conservative source theorem, include every valid `Option` input, +every Cargo profile, and both debug-assertion states: + +```text +Required_C = U_C cross O_S cross P cross D. +``` + +With applicable versioned `Option` propositions, the source proof is +parametric over this larger accepted-source region: + +```text +Covered_C = { !f on X/A/W, or f on X/A } cross O_S cross P cross D. +``` + +`h` and the exact version do not affect the source derivation. Therefore +`Required_C subset-of Covered_C` without enumerating `V x T x f x h x P x D`. + +### Expected verdicts + +- Identity of the controlling exact project support predicate: **UNRESOLVED** + (or equivalently `UNPROVED` as a policy-identification claim). +- Safe-API soundness and documented behavior over conservative union `U_C`: + **PROVED relative to `BUILD-MAP-C`**. +- The conclusion may state that either possible project promise is sound, + because both are subsets of the proved conservative domain. It must not + relabel that audit domain as the recovered exact project promise. +- `turbo` plus wasm32: outside both policies and effectively rejected by the + active `compile_error!` configuration, **PROVED relative to `BUILD-MAP-C`**. + +The non-`turbo` path uses the documented safe `unwrap_or`. The `turbo` path +returns on `None`; reaching `unwrap_unchecked` therefore establishes `Some`, +and the returned byte satisfies the same public behavior. This proof is +independent of target, `hardened`, profile, and debug assertions. + +### Atoms + +- **C1 — Both nonlinear predicates:** Reproduces both policies without + choosing precedence and identifies at least one Scarlet-only and one + Indigo-only region. +- **C2 — Conservative domain:** Takes either rigorous route: derive the exact + minimal `U_C` formula, including its conditional aarch64 clause; or choose a + transparent audit superset `S` and prove separately that + `P_Scarlet subset-of S` and `P_Indigo subset-of S`. In either route, do not + call the chosen conservative domain the unresolved exact project promise. +- **C3 — Enforced exclusion:** Recognizes that `turbo` wasm32 is excluded by + both policies and that the `compile_error!` selected by the matching `cfg` + makes the ordinary supported build interface reject it, using only the exact + accepted `BUILD-MAP-C` mappings and applicable Rust semantics. +- **C4 — Soundness certificate:** Proves the safe non-`turbo` branch and the + `None`-return/reachable-`Some` argument for the unsafe `turbo` branch, + expresses the resulting `Covered` region parametrically, proves it contains + the chosen conservative audit domain, and reports soundness **PROVED relative + to `BUILD-MAP-C`** over that domain. +- **C5 — Behavioral certificate:** Proves both branches return zero for `None` + and the contained byte for `Some`, shows the same `Covered` region contains + the chosen conservative audit domain, and reports the documented behavior + **PROVED relative to `BUILD-MAP-C`** over that domain. +- **C6 — Policy status:** Reports the controlling exact project predicate + **UNRESOLVED** and does not relabel the chosen conservative audit domain as + that promise. + +### Hard-error triggers + +A report commits a hard error if it: + +- silently selects Scarlet or Indigo, intersects them, or asserts that their + union is the recovered exact project promise; +- claims that a chosen conservative audit domain contains both published + predicates, or claims closure over it, when the stated derivation does not; + merely declining closure is an atom failure, not a hard error; +- treats `turbo` wasm32 as supported, or claims it is effectively rejected + without accounting for the selected `compile_error!`; +- treats Cargo's minimum, a hypothetical build sample, or a policy sentence as + a Rust semantic axiom; +- invents or widens a build-tool mapping beyond accepted `BUILD-MAP-C`; +- claims `PROVED` while leaving any part of the chosen conservative domain + uncovered; or +- calls the shown source `UNSOUND` or `CONTRACT-BROKEN`. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/K.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/K.md new file mode 100644 index 0000000000..4d3bcb7133 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/K.md @@ -0,0 +1,97 @@ +# Mode K Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## K — multi-issue regression + +### Exact results + +Within the exact target/profile scope declared in `REQUEST.md`, safe +implementations of `callback_index::Position` may return any `usize`. For a +one-byte slice, an implementation returning 1 gives valid safe +calls to both `read` and `write`, each of which executes its respective +out-of-bounds unchecked operation. Both safe surfaces are independently +`UNSOUND`. + +`local_proof::last` is sound. On the nonempty branch, `len > 0`, so +`index = len - 1` is defined and `index < len`; that discharges +`get_unchecked(index)`. The existing comment states none of this and does not +identify the callee obligation. The report must expose the reconstructed proof +and separately classify the implementation as proved and the proof artifact as +deficient. + +For `published_lane`, the unsafe trait contract makes valid implementations +responsible for both `INDEX < 2` and the exact `NAME`/`INDEX` relation. The shown +`High` implementation satisfies both clauses. `read` consumes only the first +clause and is sound for every valid implementation. Unknown downstream +consumers mean the unused `NAME` clause cannot be weakened from the published +1.x contract merely because `read` does not consume it. + +### Atoms + +- **K1:** For `callback_index::read`, construct the arbitrary-safe-impl witness, + prove the length-1/index-1 precondition failure and UB consequence, and report + this safe surface `UNSOUND`. +- **K2:** Independently give the same complete disposition for + `callback_index::write`; do not stop after K1. +- **K3:** For `local_proof::last`, reconstruct the nonempty -> `len > 0` -> + `len - 1 < len` derivation, connect it to the exact unchecked-index contract, + and report the implementation obligation proved. +- **K4:** Separately report the existing `local_proof::last` safety comment + materially deficient and provide replacement proof content that identifies + the unchecked-index obligation and the K3 derivation. +- **K5:** Prove that `High` satisfies both clauses of its unsafe-trait contract: + `INDEX < 2` and the exact `NAME`/`INDEX` relation. +- **K6:** Prove `published_lane::read` sound for every valid implementation of + the complete `Lane` contract by consuming its `INDEX < 2` clause. +- **K7:** Retain the `NAME` clause and explain that unknown public + implementers/consumers prevent weakening the published 1.x contract merely + because the only visible consumer uses `INDEX`. +- **K8:** Propose a repair that preserves callable read and write operations at + positions chosen by safe callers and locally couples and checks each chosen + position against the particular slice. Deleting or making the APIs + uncallable, or merely sealing or eliminating the caller-implementable trait, + does not meet that functional requirement. + +Score each atom independently. Compact shared reasoning may serve K1 and K2 +only if it explicitly names both operations and shows that the witness reaches +each contract; do not infer the second disposition from the first. + +### Hard errors + +Hard error if the report: + +- trusts arbitrary safe `Position` implementations; +- misses either safe surface while claiming complete coverage; +- calls either safe surface sound; +- rejects the correct `last` implementation because its comment is bad; +- explicitly calls the deficient comment adequate; +- relies on that comment as the proof; +- claims complete requested coverage while omitting the reconstruction; +- weakens the published trait contract from visible use alone; +- treats sealing alone as satisfying the stated functional repair; +- claims a callable caller-position API is repaired without locally coupling + the selected position to the particular slice; or +- certifies an unimplemented repair. + +A mere omitted comment-quality disposition fails K4 but is not independently a +hard error. + +### Authority + +- Rust 1.82.0 slice `get_unchecked`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked` +- Rust 1.82.0 slice `get_unchecked_mut`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut` +- Rust 1.82.0 slice `len` and `is_empty`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.len` + and + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.is_empty` +- Rust 1.82.0 Reference, arithmetic binary operators and integer overflow: + `https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html#arithmetic-and-logical-binary-operators` +- Rust 1.82.0 Reference, unsigned integer value domains: + `https://doc.rust-lang.org/1.82.0/reference/types/numeric.html#integer-types` +- Rust 1.82.0 Reference, unsafe traits: + `https://doc.rust-lang.org/1.82.0/reference/items/traits.html#unsafe-traits` +- Rust 1.82.0 Reference, unsafe-trait proof obligations: + `https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html#unsafe-traits-unsafe-trait` diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/M.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/M.md new file mode 100644 index 0000000000..dbd98585cc --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/M.md @@ -0,0 +1,171 @@ +# Mode M Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## M — Multi-release affirmative certificates + +Target: supplied M snapshot. + +Let `C_M` be every supported target times every ordinary profile. For a +function `f`, let `Calls_f` be all calls satisfying its documented safety +preconditions. The exact required domains are: + +- `Required(M-ack) = V_ack times C_M times Calls_ack`, where + `V_ack = {1.80.0, 1.80.1, 1.81.0, 1.82.0}` and every typed call is valid; +- `Required(M-store) = V_store times C_M times Calls_store`, where + `V_store = {1.80.0, 1.81.0}`; +- `Required(M-copy) = V_copy times C_M times Calls_copy`, where + `V_copy = {1.80.0, 1.80.1, 1.81.0, 1.82.0}`; and +- `Required(M-load) = V_load times C_M times Calls_load`, where + `V_load = {1.80.0, 1.80.1, 1.81.0, 1.82.0}`. + +For `store_word`, `copy_byte`, and `load_word`, the same release domain applies +separately to soundness and the documented normal-return postcondition. + +### Expected results and atoms + +- **M1 — `acknowledge` domain:** Preserve exact finite set `V_ack` and its + target/profile product; every well-typed call is in `Calls_ack` because the + API adds no safety precondition. +- **M2 — `store_word` domains:** Preserve exact finite set `V_store` and its + target/profile/valid-call product separately for soundness and the documented + normal-return postcondition. +- **M3 — `copy_byte` domains:** Preserve exact finite set `V_copy` and its + target/profile/valid-call product separately for soundness and the documented + normal-return postcondition. +- **M4 — `load_word` domains:** Preserve exact finite set `V_load` and its + target/profile/valid-call product separately for soundness and the documented + normal-return postcondition. + +For M1–M4, do not let the cutoff add or remove releases and do not substitute +an unscoped crate-wide claim for the exact products. + +- **M5 — Parametric `acknowledge` proof:** Verify that accepted general entry + `SEM-EMPTY-BLOCK-180-182` has exactly the required release, target, and + profile scope. Independently inspect the local syntax and establish that the + exact body is an empty block. For arbitrary + `(v, target, profile, call) in Required(M-ack)`, combine only those premises: + `acknowledge` is zero-parameter and unit-returning, the valid call has defined + callee evaluation and returns `()`, and the `unsafe fn` marker changes only + its static caller obligation. Thus + `Covered(M-ack) = Required(M-ack)` and source-level soundness is **PROVED + parametrically relative to `SEM-EMPTY-BLOCK-180-182`**. Keep the admission + conspicuous; the TCB entry is general semantics, not a target-specific + assertion that the function is sound. +- **M6 — `store_word` soundness partition:** The 1.80.0 `ptr::write` authority + applies exactly to the 1.80.0 case and the 1.81.0 authority to the 1.81.0 + case. For each case, the documented caller contract entails the applicable + page's alignment and write-validity preconditions. The identity + `V_store = {1.80.0} union {1.81.0}` proves exhaustiveness. Therefore + `Covered(M-store-sound) = Required(M-store-sound)` and soundness is + **PROVED** by an exact finite partition. +- **M7 — `store_word` postcondition partition:** In each exact release case, + the applicable page says that `ptr::write(dst, value)` writes the supplied + `value` to `dst` without reading or dropping the old value. The same finite + partition covers every normal-return postcondition obligation, so + `Covered(M-store-post) = Required(M-store-post)` and the documented + postcondition is **PROVED**. +- **M8 — `copy_byte` soundness under the exact TCB:** Verify the Rust 1.80.0 + `copy_nonoverlapping` safety proposition, primitive `u8` size, `u8: Copy`, + and exact `Copy` semantics base propositions. Then apply only accepted entry + `COMPAT-COPY-180-182`, with its exact release set, `T = u8`, `count = 1`, + target/profile domain, and consumer. The caller contract entails its source + and destination validity, initialization, alignment, and nonoverlap clauses; + the admitted `u8` and `Copy` facts establish the one-byte specialization and + avoid the ownership hazard for non-`Copy` values. Thus + `Covered(M-copy-sound) = Required(M-copy-sound)` and soundness is **PROVED + relative to `COMPAT-COPY-180-182`**. +- **M9 — `copy_byte` postcondition under the exact TCB:** The accepted entry + preserves the base proposition that the call copies the source byte into the + destination while leaving the source byte unchanged; `u8` has size one and + implements `Copy` with the cited copy rather than move semantics. It covers + every normal-return postcondition obligation, so + `Covered(M-copy-post) = Required(M-copy-post)` and the documented + postcondition is **PROVED relative to `COMPAT-COPY-180-182`**. +- **M10 — `load_word` soundness remainder:** In the 1.80.0 and 1.82.0 endpoint + cases, the exact `ptr::read` Safety sections plus exact-version `u32: Copy` + and `Copy`-semantics facts prove soundness from the caller contract, + including safe retention of the source value alongside the returned copy. + Thus + `Covered(M-load-sound) = {1.80.0, 1.82.0} times C_M times Calls_load`. + The 1.80.1 and 1.81.0 regions and therefore the complete `V_load` soundness + claim are **UNPROVED**. No UB witness is supplied, so `UNSOUND` does not + follow. +- **M11 — `load_word` postcondition remainder:** In the same two endpoint + cases, the applicable `ptr::read` descriptions establish that the returned + value is read from `src` while the source is left unchanged; the exact + `u32: Copy` and `Copy`-semantics facts discharge the ownership qualification. + Therefore + `Covered(M-load-post) = {1.80.0, 1.82.0} times C_M times Calls_load`. + The two interior regions and the complete postcondition claim are + **UNPROVED**. No defined wrong-result witness is supplied, so + `CONTRACT-BROKEN` does not follow. + +Scoring guidance, not a separate atom: if a report actually relies on endpoint +sampling, the cutoff, an unstated stability guarantee, backward projection, +evidence for another operation, or a widened TCB entry to fill the `ptr::read` +interior, reject that basis explicitly. For an `UNPROVED` interior, require the +smallest missing premise: a verified authority for the interior releases, an +exact applicable accepted compatibility entry, or another complete parametric +proof. A report that uses none of the listed invalid bases need not recite them. + +### M hard errors + +Any of these is a hard error: + +- reporting the complete `load_word` claim `PROVED` by endpoint sampling, + cutoff reasoning, a stability badge, unprovided interior docs, or another + operation's evidence; +- claiming `Required subset Covered` for `load_word` while omitting either + interior release; +- silently widening `COMPAT-COPY-180-182` by release, operation, type/count, + target/profile domain, proposition, or consumer; +- silently widening `SEM-EMPTY-BLOCK-180-182` by release, source shape, + proposition, or consumer; +- treating either accepted entry as authoritative Rust text rather than a + conspicuous TCB premise; +- projecting a later Rust proposition backward to an earlier release without + an exact applicable premise; or +- issuing `UNSOUND` or `CONTRACT-BROKEN` for an unresolved interior merely from + failure to prove it. + +Failure to establish a required positive regional result is an atom failure. It +is not by itself a hard error unless the report also makes one of the false +affirmative or scope-changing claims above. + +### M checked authorities + +Open each exact page and confirm the named description and Safety propositions, +including all qualifications relevant to its exact release case: + +- `https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html` + — overwrites without reading or dropping the old value; `dst` must be valid + for writes and properly aligned. +- `https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html` + — the same exact propositions for the separate 1.81.0 case. +- `https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html` + — copies `count * size_of::()` bytes; source/destination validity, + alignment, and nonoverlap requirements; base case only. +- `https://doc.rust-lang.org/1.80.0/reference/type-layout.html#primitive-data-layout` + — `u8` has size 1. +- `https://doc.rust-lang.org/1.80.0/std/primitive.u8.html#impl-Copy-for-u8` + — `u8` implements `Copy` in the compatibility base case. +- `https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html` + — exact copy rather than move semantics for `Copy` values in the base and + 1.80.0 endpoint cases. +- `https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html` + — reads without moving, leaves source unchanged, and requires read validity, + alignment, and initialization for this non-ZST. +- `https://doc.rust-lang.org/1.80.0/std/primitive.u32.html#impl-Copy-for-u32` + — `u32` implements `Copy` in the 1.80.0 endpoint case. +- `https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html` + — the same named propositions for the separate endpoint case. +- `https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#impl-Copy-for-u32` + — `u32` implements `Copy` in the 1.82.0 endpoint case. +- `https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html` + — exact copy rather than move semantics for `Copy` values in the 1.82.0 + endpoint case. + +The `acknowledge` and compatibility results additionally consume the two exact +accepted propositions in target file `TCB.md`; that file is not Rust authority +and both admissions must remain identified as part of the conditional TCB. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/Q.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/Q.md new file mode 100644 index 0000000000..fcec5e9489 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/Q.md @@ -0,0 +1,76 @@ +# Mode Q Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## Q — Existential refutation and missing unsafe dependency + +Target: supplied Q snapshot. + +Let `C_Q` be Rust 1.80.0 times every target on which the exact source and pinned +dependency resolve times every ordinary profile. + +- `Required(Q-force-sound) = C_Q` times the one well-typed safe call + `force_unreachable()`. +- `Required(Q-delegate-sound) = C_Q` times every call + `delegated_nonzero(value)` whose caller satisfies `value != 0`. + +### Expected results and atoms + +- **Q1 — Domain and claim separation:** State both valid-use domains and audit + the two APIs independently. In particular, the `unsafe fn`'s documented + nonzero precondition restricts its valid uses; it does not restrict the safe + `force_unreachable` API. +- **Q2 — Complete UB certificate:** For `force_unreachable`, give all four + links explicitly or by an equivalently exact chain: + 1. `force_unreachable()` is a well-typed, valid safe call in `Required`; + 2. the only body path reaches the executed + `std::hint::unreachable_unchecked()` call; + 3. that call requires its site not to be reached, and that exact proposition + is false for this call; and + 4. Rust 1.80.0 std documentation says reaching the function is undefined + behavior. +- **Q3 — Existential closure:** Report `Q-force-sound` as **UNSOUND**. The one + complete in-scope witness refutes the universal soundness claim; do not + weaken it to `UNPROVED` because no proof covering every safe call was built. + The witness exists in every `C_Q` configuration, but that stronger fact is + not required for the existential verdict. +- **Q4 — Exact missing premise:** For `delegated_nonzero`, recognize that a + valid caller supplies the dependency API's nonzero caller-side precondition, + but the selected unsafe dependency's body is absent and no accepted + `UNSAFE-DEP` entry establishes that it honors its contract. The exact pin and + caller-facing documentation establish the logical package coordinate and the + propagated obligation, not authenticated implementation identity or + correctness. For the soundness claim actually requested here, the smallest + missing proposition is that every valid nonzero call to the selected + `peer-unsafe-dep` 3.4.5 implementation is free of UB over this exact domain. +- **Q5 — No fabricated witness or silent trust:** Report + `Q-delegate-sound` as **UNPROVED**, not `PROVED` and not `UNSOUND`. From the + submitted packet, claim-level `Covered(Q-delegate-sound)` contains no case: + the dependency-soundness obligation is unresolved for all valid inputs. + There is also no proved execution of the unavailable body, so no UB witness + may be invented. State that recursive source audit or an explicitly accepted + exact `UNSAFE-DEP` entry would be required. + +### Q hard errors + +Any of these is a hard error: + +- calling `force_unreachable` sound or only `UNPROVED` after stating all four + certificate links; +- issuing `UNSOUND` for it without a valid call, reachability, the false exact + safety proposition, and the authoritative UB consequence; +- silently trusting the unavailable unsafe dependency, treating the exact pin + as implementation proof, or reporting `delegated_nonzero` `PROVED`; +- fabricating dependency body behavior or a UB execution and reporting + `delegated_nonzero` `UNSOUND`; or +- treating the unsafe dependency's documented caller precondition as though + violating its prose were itself an executed UB event. + +### Q checked authority + +- `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` + — exact proposition consumed: reaching + `unreachable_unchecked` is undefined behavior; applicability Rust 1.80.0, + without a target/profile qualification relevant to this fixture. + +No Rust authority is asserted for the absent dependency implementation. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/R.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/R.md new file mode 100644 index 0000000000..17b4a752e1 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/R.md @@ -0,0 +1,67 @@ +# Mode R Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## R — abstraction redesign + +### Exact result + +Within the exact target/profile scope declared in `REQUEST.md`, `Required` +includes every safe downstream implementation of `Slot` and safe instantiation +of `increment`, not only the crate-owned `Tail`. A safe +implementation can return 2. Calling `increment::(&mut [0, 0])` then +executes `get_unchecked_mut(2)` on a length-2 slice. The Rust 1.82.0 contract +requires an in-bounds index and says an out-of-bounds call is UB even if the +result is not used. The current safe API is therefore `UNSOUND`. + +The requested behavior needs neither generic pointer/index metadata nor an +unsafe abstraction. A preferred redesign is a nongeneric safe function that +updates `pair[1]` (or an equivalent checked safe specialization). It eliminates +the caller implementation capability and the unsafe block. Making `Slot` +unsafe, adding a prose rule to the safe trait, or preserving the generic +abstraction by default is less parsimonious because no downstream generic use +is required. The proposal does not affect the current verdict and needs a fresh +audit after implementation. + +### Atoms + +- **R1:** Quantify over arbitrary safe `Slot` implementations; explicitly + construct or describe one returning 2 and the safe call using it. +- **R2:** Prove reachability of `get_unchecked_mut(2)`, falsity of its in-bounds + precondition for length 2, and the applicable Rust 1.82 UB consequence. +- **R3:** Report the current safe API `UNSOUND`, independently of design intent + and proposals. +- **R4:** Extract the exact minimum required behavior: wrapping increment of + element 1 for the owned use, with no generic downstream implementation need. +- **R5:** Propose a nongeneric safe specialization that implements the required + wrapping increment of element 1 using checked indexing (or an exactly + behavior-equivalent safe operation), with no caller-controlled + implementation or index capability. +- **R6:** Explain that removing the unpublished generic trait/API is an + authorized contract delta with no promised downstream migration burden. +- **R7:** Keep the redesign conditional and require implementation plus fresh + audit; do not use it to narrow or alter the current `UNSOUND` result. + +Merely making the trait unsafe does not pass R5: it preserves a caller +implementation capability that the supplied requirement expressly does not +need. + +### Hard errors + +Hard error if the report: + +- trusts `Tail` as the only safe implementation; +- treats safe trait prose as a caller obligation; +- calls the current API sound or merely `UNPROVED` after closing the witness; +- certifies an unimplemented redesign; or +- lets design intent narrow the current contract. + +### Authority + +- Rust 1.82.0 slice `get_unchecked_mut`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut` + — the index must be in bounds; out-of-bounds calls are UB even if the + resulting reference is unused. +- Rust 1.82.0 `u32::wrapping_add`: + `https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#method.wrapping_add` + — wrapping modular addition supplies the requested update semantics. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/S.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/S.md new file mode 100644 index 0000000000..b7dd209431 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/S.md @@ -0,0 +1,111 @@ +# Mode S Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## S — Symbolic Interval and Parametric Superset + +### Intended theorem domain + +Let: + +```text +R_S = { r | StableRustRelease(r) and 1.84.0 <= r <= 1.86.0 } +T_S = { x86_64-unknown-linux-gnu, + aarch64-apple-darwin, + wasm32-unknown-unknown } +F_S = { telemetry-off, telemetry-on } +O_S = { None } union { Some(b) | b is any u8 } +``` + +The exact requested case predicate is: + +```text +Required_S(r, t, f, p, d, o) + iff r in R_S and t in T_S and f in F_S and p in P and d in D and o in O_S. +``` + +This predicate is symbolic. It is not the CI matrix or the Cargo minimum. It +contains non-`.0` stable releases such as Rust 1.84.1 and 1.85.1, but a report +may preserve the exact symbolic predicate without enumerating any members. + +Define `Q_Option(r)` to mean that the two required `Option` propositions have +been established for release `r` by either of two admissible bases: + +1. the Rust 1.84.0 base authorities plus accepted entry + `COMPAT-OPTION-184-186` over its exact region; or +2. an exact finite partition that proves the released members of `R_S` are + `{1.84.0, 1.84.1, 1.85.0, 1.85.1, 1.86.0}` and verifies the two exact + versioned pages for every member. + +The intended proof cases are: + +```text +Covered_S = { (r,t,f,p,d,o) | Q_Option(r), and t/f/p/d/o are otherwise arbitrary }. +``` + +The source derivation is parametric in `t`, `f`, `p`, and `d`; it need not and +should not be expanded into their Cartesian product. Applicability still must +establish `R_S subset-of {r | Q_Option(r)}`. Either admissible basis above can +do so. A merely report-authored compatibility proposal, generic stability +assertion, endpoint sampling, or incomplete release partition cannot. + +### Expected verdicts + +- Safe-API soundness over `Required_S`: **PROVED**, either from the exact + finite authoritative partition or relative to `COMPAT-OPTION-184-186`. +- The documented `None -> 0` and `Some(b) -> b` behavior over `Required_S`: + **PROVED** on the same basis and with the TCB qualification whenever that + entry is consumed. +- Configuration closure: **PROVED** on the chosen basis, because + `Required_S subset-of Covered_S`. + +The local proof is: `is_none()` returning true takes the `return 0` branch. If +the unsafe call is reached, `is_none()` returned false; since `Option` has only +`None` and `Some`, the unchanged, still-owned value is `Some`. This discharges +`unwrap_unchecked`'s exact precondition, and its return contract supplies the +contained byte. No target, feature, profile, or debug-assertion fact enters the +derivation. + +### Atoms + +- **S1 — Lossless domain recovery:** States `R_S` as the closed symbolic + stable-release predicate and keeps the three targets and both feature states. + It does not substitute Cargo's lower bound or CI's rows for the policy. Exact + symbolic notation passes without naming any particular patch release. +- **S2 — Premise applicability:** Uses one complete admissible evidence basis: + either verifies the 1.84.0 base, applies only fixture-supplied accepted entry + `COMPAT-OPTION-184-186`, and keeps that admission conspicuous; or proves the + exact five-member release partition and verifies both exact pages for every + member. A report-authored premise, partial partition, or mixed gap does not + pass. +- **S3 — Local safety proof:** Gives the reachability/variant argument that + reaching `unwrap_unchecked` implies the unchanged, still-owned value is + `Some`, thereby discharging the exact unsafe precondition. +- **S4 — Local behavioral proof:** Shows that `None` returns zero and `Some(b)` + returns `b`, using the applicable exact `Option` propositions. +- **S5 — Parametric configuration coverage:** Shows that the local derivation + is independent of target, feature, profile, and debug assertions and covers + their complete product without Cartesian enumeration. +- **S6 — Soundness closure and verdict:** Establishes + `Required_S subset-of Covered_S` for the soundness theorem and reports + soundness `PROVED` over the exact policy domain, conspicuously qualified by + `COMPAT-OPTION-184-186` if and only if that route was consumed. +- **S7 — Behavioral closure and verdict:** Establishes + `Required_S subset-of Covered_S` for the documented result theorem and + reports behavior `PROVED` over the exact policy domain, conspicuously + qualified by `COMPAT-OPTION-184-186` if and only if that route was consumed. + +### Hard-error triggers + +A report commits a hard error if it: + +- claims full closure or an affirmative whole-domain result after contracting + the release predicate to `.0` releases, CI rows, or the Cargo minimum; +- calls endpoints, a toolchain pin, or sparse documentation a proof of every + interior release and then claims `PROVED`; +- silently invents or accepts a report-authored compatibility premise, or hides + the fixture TCB qualification after consuming that entry; +- claims closure while leaving a `Required_S` region outside its proof; +- finds `None` reachable at `unwrap_unchecked`; or +- gives an unqualified `UNSOUND` or `CONTRACT-BROKEN` verdict for the shown + source. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/SCORER.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/SCORER.md new file mode 100644 index 0000000000..9f96351959 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/SCORER.md @@ -0,0 +1,72 @@ +# Frozen Common Blind-Scoring Rules + +Score explicit propositions, not keywords, terminology, organization, or a +preferred proof style. Accept any equivalently exact derivation. Do not infer a +missing material premise from vague shorthand. + +For each report A–J: + +1. Mark every mode atom `PASS` or `FAIL`; every material clause of an atom must + hold for `PASS`. +2. Record every independently applicable hard error. +3. Separately decide proposal laundering, a TCB/authority defect, and a visible + source-scope defect. The runner, not the scorer, decides the mechanical word + budget and records operational scope deviations. +4. Cite compact report evidence for every decision. +5. Record genuine rubric ambiguity and novel findings without creating a new + confirmatory atom. + +`Required` is the exact product of release/configuration scope, valid uses, and +the domain of the theorem being judged. `Covered` contains a case only when +every obligation has a complete applicable derivation. A universal positive +result requires an explicit or readily checkable `Required subset-of Covered` +derivation. An existential `UNSOUND` result instead requires a valid in-scope +safe use, reachability, falsity of the exact safety proposition, and an +applicable authoritative UB consequence. + +Where a mode uses these symbols, `P` ranges over every Cargo profile supported +by the target and `D` ranges over both states of debug assertions. + +A missed atom is not automatically a hard error. Apply a hard error only when +the report makes the listed false affirmative claim or triggers another exact +mode/global rule. Additional correct regional detail is harmless. + +A material Rust premise passes only when the report verifies applicable, +version-matched Rust Reference or standard-library text, or correctly leaves +the proposition unresolved. The URL allowlist grants page identity, not a +proposition. Cargo documentation is not a Rust semantic axiom. A target TCB +entry is not Rust authority; accept it only at its exact accepted identity, +proposition, consumer, and region, with conspicuous qualification where used. + +Mark `proposal_laundering` present exactly when a report uses an unimplemented +proposal as evidence that the supplied current artifact is sound, meets a +current postcondition, or has narrower obligations, or when it certifies the +proposal without implementation and fresh audit. Record G10 and any applicable +mode hard-error ID independently; this flag does not replace them. + +Global hard errors have IDs G1 through G12 in the list order below. A mode's +own hard errors have IDs `H1`, `H2`, and so on in the list order +under that rubric's **Hard errors** heading. Record an exact applicable ID, not +a report-authored category name. Global hard errors are: + +- claiming full `PROVED` after contracting or failing to justify `Required`; +- claiming positive closure without `Required subset-of Covered`; +- inventing policy precedence or calling a conservative audit domain the + project's promise; +- treating CI, endpoints, sparse documentation, a cutoff, or a stability badge + as interval coverage; +- issuing `UNSOUND` without all four existential certificate links; +- issuing only `UNPROVED` after explicitly establishing all four links; +- issuing `CONTRACT-BROKEN` using only an execution that contains UB; +- claiming an observation remains guaranteed because it occurs before UB; +- silently trusting a third-party unsafe implementation or caller-controlled + safe behavior; +- certifying an unimplemented design or using it to narrow the current-artifact + obligation; +- necessarily relying on unchecked, invalid, or inapplicable authority; or +- evidence that the report agent inspected prohibited oracle, sibling, + condition-map, prior-report, or evaluator material. + +Do not identify, cluster, or speculate about generating conditions. Do not +inspect enclosing or sibling directories, other modes, manifests, skill +packages, condition maps, prior scores, or another scorer's output. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/W.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/W.md new file mode 100644 index 0000000000..37a9acfe1a --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/W.md @@ -0,0 +1,66 @@ +# Mode W Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## W — Whole-execution UB and independent behavioral refutation + +Target: supplied W snapshot. + +Let `C_W` be Rust 1.80.0 times every target on which this exact source and the +used Rust 1.80.0 standard-library item exist times every ordinary profile, and +let `I = {0, ..., 255}`. The claims are: + +- `Required(W-sound) = C_W times I`; +- `Required(W-panic) = C_W times {0}`; and +- `Required(W-return) = C_W times I`; its theorem is: for every UB-free defined + execution in this domain, if that execution returns normally, its result + equals its input. + +### Expected results and atoms + +- **W1 — Soundness witness:** `classify(0)` is a valid safe call, takes the + zero arm, evaluates the local statements, and reaches + `unreachable_unchecked`. Its unreachable-site proposition is false; the + applicable 1.80.0 std contract entails UB. Report `W-sound` **UNSOUND**. +- **W2 — Whole-execution classification:** The input-zero execution as a whole + contains UB. Neither its apparent prefix computation nor the source-level + fact that no panic operation appears before the unsafe call remains a + guaranteed observation from a defined execution. It therefore cannot prove + a UB-free failure of the panic postcondition. With no independent UB-free + input-zero execution or equivalent proof, report `W-panic` **UNPROVED**, not + `CONTRACT-BROKEN` and not `PROVED`. Its claim-level `Covered` set is empty on + the submitted evidence. +- **W3 — Independent defined witness:** `classify(1)` takes the `1 => 2` arm, + executes no unsafe operation, returns normally, and returns `2` rather than + the promised input value `1`. That complete execution is UB-free and proves + `W-return` **CONTRACT-BROKEN**. + +Scoring guidance, not a separate atom: keep the three executions/theorems +independent. The input-zero UB witness cannot establish failure of the panic +promise, while the independent, defined input-one failure remains usable and +does not alter the soundness witness. + +### W hard errors + +Any of these is a hard error: + +- calling the safe API sound, reporting only `UNPROVED` after explicitly + establishing every input-zero UB-certificate link, or issuing `UNSOUND` + without those links; +- using the UB-containing input-zero execution, an apparent observation before + its UB, or the absence of an earlier panic to issue `CONTRACT-BROKEN`; +- claiming source observations remain guaranteed merely because they occur + before the unsafe call; +- falsely treating the independent input-one refutation as UB-containing, or + claiming complete behavioral coverage while omitting it. + +### W checked authorities + +- `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` + — reaching the function is undefined behavior. +- `https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html` + — controlling Rust 1.80.0 Reference section for what constitutes undefined + behavior. Verify that the final evaluator instructions do not attribute to + this page prose it does not contain; whole-execution non-observability is the + logical consequence of classifying the execution as undefined, not a claimed + quotation from a “time travel” subsection. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/X.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/X.md new file mode 100644 index 0000000000..51a1f0c935 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/X.md @@ -0,0 +1,145 @@ +# Mode X Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## X — Simultaneous Feature, Target, and Allocator + +### Intended theorem domain + +Let: + +```text +R_X = { Rust 1.85.1 } +T_X = { X = x86_64-unknown-linux-gnu, + A = aarch64-unknown-linux-gnu, + W = wasm32-unknown-unknown } +B_X = { burst-off, burst-on } +L_X = { system, arena } +I_X = { every u8 } +``` + +The exact supported configuration predicate is: + +```text +Config_X(t,b,l) + iff t in T_X and b in B_X and l in L_X and not (t = W and l = arena). + +Required_X = R_X cross Config_X cross P cross D cross I_X. +``` + +In particular, this simultaneous cell is supported: + +```text +Q_X = (target = A, burst = on, allocator = arena). +``` + +The only target/allocator exclusion is `target = W and allocator = arena`, +for either feature state. It is distinct from `Q_X`. + +The build script maps the accepted `FIXTURE_ALLOCATOR` value to exactly one +`fixture_allocator` option; the Rust conditional-compilation rules then select +the corresponding source. For positive soundness bookkeeping, the +implementation closes exactly these call cases: + +```text +Covered_X_sound = { case in Required_X | not Q_X or value != 0 }. +``` + +Outside `Q_X`, the explicit zero check panics for zero; after that check, +`new_unchecked(value)` meets its precondition. Inside `Q_X`, the unchecked +constructor is reached without that check and meets its precondition only for +nonzero values. Consequently `Required_X` is not a subset of +`Covered_X_sound`. + +### Expected verdicts + +- Safe-API soundness over `Required_X`: **UNSOUND relative to accepted + `BUILD-MAP-X`**. +- Whole-domain postcondition “zero panics”: **UNPROVED**, not + `CONTRACT-BROKEN`, from the known counterexample, because that execution + contains UB; the configuration classification is relative to `BUILD-MAP-X`. +- Soundness outside `Q_X`, and for nonzero inputs inside `Q_X`: **PROVED + relative to `BUILD-MAP-X`** with the version-matched contracts. +- The documented zero-input panic guarantee outside `Q_X`: **PROVED relative + to `BUILD-MAP-X`**. +- The wasm32/arena pair: genuinely excluded and rejected; it is not the UB + witness. Effective rejection is **PROVED relative to `BUILD-MAP-X`**. + +The closing witness is the fully safe call `lane_id(0)` on Rust 1.85.1 for +`aarch64-unknown-linux-gnu`, with `burst` enabled and the accepted `arena` +allocator selection. The build output and all three true `cfg` conjuncts make +the unsafe branch reachable. It calls `NonZeroU8::new_unchecked(0)`, whose +exact safety requirement is false and whose applicable documentation states +the UB consequence. One supported witness closes `UNSOUND`. + +### Atoms + +- **X1 — Complete cross-axis domain:** Recovers the target, feature, and + allocator axes together; includes `Q_X`; and records only wasm32/arena as the + policy exclusion. +- **X2 — Selector partition:** Recovers the complete environment-input + partition: omitted, Unicode `system`, Unicode `arena`, every other Unicode + value, and every non-Unicode value. +- **X3 — Accepted selector mapping:** Shows that omitted and explicit `system` + select `system`, while explicit `arena` selects `arena`, on build attempts + whose directive writes succeed. +- **X4 — Rejected selector behavior:** On attempts where the preceding + directive writes succeed, shows that every other Unicode value and every + non-Unicode value reaches a panic arm and, under `BUILD-MAP-X`, produces no + library compilation. It distinguishes that policy rejection from an earlier + infrastructure write failure, which also produces no compilation but never + reaches selector handling. +- **X5 — Allocator-cfg cardinality:** Shows that every successful accepted + selector path emits exactly one `fixture_allocator` cfg and that its value is + the value selected in X3. +- **X6 — Check-cfg directive:** After the preceding rerun write succeeds, shows + that the script attempts the check-cfg write before selector handling and + that, when this write succeeds, it registers exactly `system` and `arena` as + expected values without selecting either value. Failure of either write + follows the infrastructure no-compilation branch. +- **X7 — Rerun directive:** Shows that the script unconditionally attempts the + environment-change rerun write and that, when the write succeeds, + `BUILD-MAP-X` causes Cargo to rerun the script when `FIXTURE_ALLOCATOR` + changes. A failed write follows the infrastructure no-compilation branch. +- **X8 — Generated-configuration reachability:** Follows accepted `arena` + selection through `build.rs` to `fixture_allocator="arena"` and then applies + the feature, target, and allocator cfg conjuncts to reach `Q_X`. It does not + audit only handwritten library source in isolation. +- **X9 — Existential UB certificate:** Gives the supported safe + `lane_id(0)` call in `Q_X`, reaches `new_unchecked(0)`, states the exact + false nonzero proposition and applicable UB consequence, and reports + `UNSOUND`. +- **X10 — Quantifier-sensitive behavior:** Calls the universal zero-input panic + guarantee `UNPROVED`, not `CONTRACT-BROKEN`, because the known failing + execution has UB. +- **X11 — Remainder soundness:** Proves soundness outside `Q_X` by following the + explicit zero guard and applying `new_unchecked` only after it establishes a + nonzero value, and proves nonzero inputs sound inside `Q_X`, relative to the + exact build mapping. +- **X12 — Regional panic theorem:** Proves that zero takes the explicit + `panic!` branch in every configuration outside `Q_X`, relative to the exact + build mapping. It asserts no undocumented nonzero return-value postcondition. +- **X13 — Genuine exclusion:** Shows both that policy excludes wasm32/arena and + that the active `compile_error!` rejects that pair after allocator cfg + generation under `BUILD-MAP-X`. It does not use this excluded pair as the + witness or generalize the exclusion to aarch64/arena. + +### Hard-error triggers + +A report commits a hard error if it: + +- omits feature, target, or allocator and then uses the contracted conjunction + to claim full-domain closure/soundness, or treats the supported `Q_X` cell as + excluded; +- ignores build-script output and consequently declares the unsafe branch + unreachable; +- invents, hides, or widens a build-tool/target mapping beyond accepted + `BUILD-MAP-X`; +- uses the excluded wasm32/arena pair as a valid soundness counterexample; +- calls the current safe API sound or stops at `UNPROVED` despite establishing + the complete supported `lane_id(0)` UB witness; +- calls the zero-panic promise `CONTRACT-BROKEN` based on the UB-containing + execution, or claims observations before/after that UB remain guaranteed; +- claims the policy exclusion alone makes a reachable safe API sound without + checking effective rejection; or +- certifies an unimplemented repair instead of the supplied source snapshot. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/adjudication.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/adjudication.schema.json new file mode 100644 index 0000000000..a9ceffde2b --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/adjudication.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-blind-adjudication-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "decisions", "ambiguities"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "decisions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "field", "decision", "evidence"], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "field": { + "type": "string", + "pattern": "^(atom:[SCXQWMRK][1-9][0-9]*|hard_error:(G[1-9][0-9]*|[SCXQWMRK]H[1-9][0-9]*)|proposal_laundering|tcb_authority_defect|visible_scope_defect|novel:s[12]:N[1-9][0-9]*)$" + }, + "decision": { + "enum": ["PASS", "FAIL", "PRESENT", "ABSENT"] + }, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + }, + "ambiguities": { + "type": "array", + "items": {"type": "string", "pattern": "\\S"} + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/disagreements.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/disagreements.schema.json new file mode 100644 index 0000000000..ae7b277ff5 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/disagreements.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-v3-targeted-disagreements-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "cells"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "cells": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "field", "s1", "s2"], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "field": {"type": "string", "minLength": 1}, + "s1": {"$ref": "#/$defs/position"}, + "s2": {"$ref": "#/$defs/position"} + } + } + } + }, + "$defs": { + "position": { + "type": "object", + "additionalProperties": false, + "required": ["decision", "evidence"], + "properties": { + "decision": {"enum": ["PASS", "FAIL", "PRESENT", "ABSENT"]}, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/event.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/event.schema.json new file mode 100644 index 0000000000..a49542dff4 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/event.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-v3-targeted-event-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "sequence", "previous_event_sha256", "time_utc", "phase", "event", "details"], + "properties": { + "schema_version": {"const": 1}, + "sequence": {"type": "integer", "minimum": 1}, + "previous_event_sha256": { + "oneOf": [ + {"type": "null"}, + {"type": "string", "pattern": "^[0-9a-f]{64}$"} + ] + }, + "time_utc": {"type": "string", "format": "date-time"}, + "phase": { + "enum": ["freeze", "collection", "scoring", "adjudication", "unblinding", "result"] + }, + "event": {"type": "string", "minLength": 1}, + "run_id": {"type": "string"}, + "attempt": {"type": "integer", "minimum": 1}, + "agent_id": {"type": "string"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "details": {"type": "object"} + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/final.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/final.schema.json new file mode 100644 index 0000000000..53d7343921 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/final.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-v3-targeted-final-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "reports", "confirmed_novel_findings"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "reports": { + "type": "array", + "minItems": 10, + "maxItems": 10, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "atoms", "hard_errors", "proposal_laundering", "tcb_authority_defect", "visible_scope_defect"], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "atoms": {"type": "object"}, + "hard_errors": {"type": "array", "items": {"type": "string"}}, + "proposal_laundering": {"type": "boolean"}, + "tcb_authority_defect": {"type": "boolean"}, + "visible_scope_defect": {"type": "boolean"} + } + } + }, + "confirmed_novel_findings": { + "type": "array", + "items": {"type": "object"} + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/score.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/score.schema.json new file mode 100644 index 0000000000..b9509706d0 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/score.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-blind-score-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "scorer_id", "reports", "ambiguities"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "scorer_id": {"enum": ["s1", "s2"]}, + "reports": { + "type": "array", + "minItems": 10, + "maxItems": 10, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "label", + "atoms", + "hard_errors", + "proposal_laundering", + "tcb_authority_defect", + "visible_scope_defect", + "novel_findings" + ], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "atoms": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "decision", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^[SCXQWMRK][1-9][0-9]*$"}, + "decision": {"enum": ["PASS", "FAIL"]}, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + }, + "hard_errors": { + "type": "array", + "items": {"$ref": "#/$defs/hard_error"} + }, + "proposal_laundering": {"$ref": "#/$defs/flag"}, + "tcb_authority_defect": {"$ref": "#/$defs/flag"}, + "visible_scope_defect": {"$ref": "#/$defs/flag"}, + "novel_findings": { + "type": "array", + "items": {"$ref": "#/$defs/novel_finding"} + } + } + } + }, + "ambiguities": { + "type": "array", + "items": {"type": "string", "pattern": "\\S"} + } + }, + "$defs": { + "hard_error": { + "type": "object", + "additionalProperties": false, + "required": ["id", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^(G[1-9][0-9]*|[SCXQWMRK]H[1-9][0-9]*)$"}, + "evidence": {"type": "string", "pattern": "\\S"} + } + }, + "novel_finding": { + "type": "object", + "additionalProperties": false, + "required": ["id", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^N[1-9][0-9]*$"}, + "evidence": {"type": "string", "pattern": "\\S"} + } + }, + "flag": { + "type": "object", + "additionalProperties": false, + "required": ["present", "evidence"], + "properties": { + "present": {"type": "boolean"}, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/operations.lock b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/operations.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/prepare.py b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/prepare.py new file mode 100644 index 0000000000..23a3d8b343 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/prepare.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Generate and verify the frozen V3 targeted-evaluation artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + + +RUN = Path(__file__).resolve().parent +FREEZE = RUN / "freeze" +SEALED = RUN / "sealed" +EVALS = RUN.parent.parent + +MODES = ("S", "C", "X", "Q", "W", "M", "R", "K") +CONDITIONS = ("v3", "v2") +REPLICATES = range(1, 6) +ATOM_COUNTS = {"S": 7, "C": 6, "X": 13, "Q": 5, "W": 3, "M": 11, "R": 7, "K": 8} +WORD_CAPS = {"S": 1800, "C": 1800, "X": 2400, "Q": 1800, "W": 1800, "M": 1800, "R": 1800, "K": 2200} + +PACKAGES = { + "v3": { + "path": "frozen-packages/668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf", + "tree": "668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf", + "skill": "0e23f7747cc63014bade7543efaf745e7e9a7e5d6dee2a48c602ef7a3eba091e", + }, + "v2": { + "path": "frozen-packages/40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897", + "tree": "40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897", + "skill": "a0a75ef8a14497aa78b50b459981097ee99605c57fec95c637cf59aaa20fe766", + }, +} + +TARGETS = { + "S": ("s_symbolic", "28ecc523e15b914a187814ab2752c0d85996948a552c62f970d8484bc6ed467a"), + "C": ("c_conflict", "065c3cfc032af93e7576e17e49322826c4a379a707870175b16c2663d1e8e4e0"), + "X": ("x_cross", "25b4efef689601f3b5983bf6b914bd367153d0b21a6f1f4d40de50c5d412afa7"), + "Q": ("q_quantifiers", "c0a4c43373a159cb38d08724af8b02187b249ab6a73f7e1b10b2276f38b0cb5a"), + "W": ("w_whole_execution", "b27b95fbc9ffa9d335bb6b4614a9f227a5798122fca79b42cf53c9b106a6aff6"), + "M": ("m_multirelease", "b269cf068196d1c06b87be6bcded494827e7ac8a2cb6debf1eb0f0f5d0388479"), + "R": ("r_redesign", "6d1a41909b012484d71f94d194071a7ebbb6773b7596db88127ca2a195b70ffc"), + "K": ("k_regression", "ca272e524b36a892e25f6169631a184ff764026451eff2f6ac4ab8e9d5e87ea2"), +} + +RUBRIC_RANGES = { + "S": ("domain.md", "## S —", "## C —"), + "C": ("domain.md", "## C —", "## X —"), + "X": ("domain.md", "## X —", "## Exact Authority"), + "Q": ("verdict.md", "## Q —", "## W —"), + "W": ("verdict.md", "## W —", "## M —"), + "M": ("verdict.md", "## M —", None), + "R": ("controls.md", "## R —", "## K —"), + "K": ("controls.md", "## K —", None), +} + +AUTHORITY_RANGES = { + "S": ("domain.md", "### S authorities", "### C authorities"), + "C": ("domain.md", "### C authorities", "### X authorities"), + "X": ("domain.md", "### X authorities", None), + "Q": ("verdict.md", "## Q —", "## W —"), + "W": ("verdict.md", "## W —", "## M —"), + "M": ("verdict.md", "## M —", None), + "R": ("controls.md", "## R —", "## K —"), + "K": ("controls.md", "## K —", None), +} + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def keyed(tag: str, seed: str, value: str) -> str: + return sha256(tag.encode() + b"\0" + bytes.fromhex(seed) + b"\0" + value.encode()) + + +def read_seeds() -> dict[str, str]: + data = json.loads((SEALED / "seeds.json").read_text()) + expected = {"condition", "schedule", "blind", "presentation", "scorer"} + if set(data) != expected: + raise ValueError("unexpected seed keys") + for name, value in data.items(): + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError(f"invalid {name} seed") + if value == "0" * 64: + raise ValueError(f"zero {name} seed") + if len(set(data.values())) != len(data): + raise ValueError("randomization seeds are not distinct") + return data + + +def extract_rubric(mode: str) -> str: + filename, start_marker, end_marker = RUBRIC_RANGES[mode] + source = (FREEZE / "oracle" / filename).read_text() + start = source.index(start_marker) + end = source.index(end_marker, start + len(start_marker)) if end_marker else len(source) + body = source[start:end].rstrip() + "\n" + atoms = re.findall(rf"^- \*\*{mode}[0-9]+\b", body, flags=re.MULTILINE) + if len(atoms) != ATOM_COUNTS[mode]: + raise ValueError(f"{mode}: expected {ATOM_COUNTS[mode]} atoms, found {len(atoms)}") + return ( + f"# Mode {mode} Frozen Blind-Scoring Rubric\n\n" + "> **Evaluator-only material. Never expose this file to a report agent.**\n\n" + + body + ) + + +def generated_files(seeds: dict[str, str]) -> dict[Path, str]: + outputs: dict[Path, str] = {} + + for mode in MODES: + outputs[FREEZE / "rubrics" / f"{mode}.md"] = extract_rubric(mode) + + condition_order = sorted(CONDITIONS, key=lambda role: (keyed("condition-v1", seeds["condition"], role), role)) + condition_labels = {role: f"c{index}" for index, role in enumerate(condition_order)} + + mode_order = sorted(MODES, key=lambda mode: (keyed("mode-label-v1", seeds["schedule"], mode), mode)) + mode_labels = {mode: f"m{index}" for index, mode in enumerate(mode_order)} + + condition_rows = ["condition_label\trole\tpackage_path\ttree_sha256\tskill_sha256"] + for role in condition_order: + package = PACKAGES[role] + condition_rows.append( + "\t".join((condition_labels[role], role, package["path"], package["tree"], package["skill"])) + ) + outputs[SEALED / "condition-map.tsv"] = "\n".join(condition_rows) + "\n" + + target_rows = ["target_label\tmode\tsource_path\ttree_sha256\tword_cap"] + for mode in mode_order: + directory, digest = TARGETS[mode] + target_rows.append( + f"{mode_labels[mode]}\t{mode}\tfixtures/v3-targeted/{directory}\t{digest}\t{WORD_CAPS[mode]}" + ) + outputs[SEALED / "target-map.tsv"] = "\n".join(target_rows) + "\n" + + wave_order = sorted(REPLICATES, key=lambda rep: (keyed("wave-v1", seeds["schedule"], str(rep)), rep)) + schedule_rows = ["run_id\tcell_id\twave\ttarget_label\tcondition_label\treplicate"] + schedule: list[tuple[str, str, int, str, str, int, str, str]] = [] + run_number = 1 + for wave_index, replicate in enumerate(wave_order, start=1): + cells = [(mode, role, replicate) for mode in MODES for role in CONDITIONS] + cells.sort( + key=lambda cell: ( + keyed("schedule-v1", seeds["schedule"], f"{cell[0]}|{cell[1]}|{cell[2]}"), + cell, + ) + ) + for mode, role, rep in cells: + canonical = f"{mode}|{role}|{rep}" + cell_id = keyed("cell-v1", seeds["schedule"], canonical)[:32] + run_id = f"r{run_number:03d}" + schedule_rows.append( + f"{run_id}\t{cell_id}\t{wave_index}\t{mode_labels[mode]}\t{condition_labels[role]}\t{rep}" + ) + schedule.append((run_id, cell_id, wave_index, mode, role, rep, mode_labels[mode], condition_labels[role])) + run_number += 1 + if len(schedule) != 80 or len({row[1] for row in schedule}) != 80: + raise ValueError("schedule does not contain 80 unique cells") + outputs[SEALED / "launch-schedule.tsv"] = "\n".join(schedule_rows) + "\n" + + blind_rows = ["mode\tlabel\trun_id"] + for mode in MODES: + run_ids = [row[0] for row in schedule if row[3] == mode] + run_ids.sort(key=lambda run_id: (keyed("blind-v1", seeds["blind"], f"{mode}|{run_id}"), run_id)) + for index, run_id in enumerate(run_ids): + blind_rows.append(f"{mode}\t{chr(ord('A') + index)}\t{run_id}") + outputs[SEALED / "blind-map.tsv"] = "\n".join(blind_rows) + "\n" + + presentation_rows = ["claim\tlabels_in_order"] + for mode in MODES: + for scorer in ("s1", "s2"): + labels = [chr(ord("A") + index) for index in range(10)] + labels.sort( + key=lambda label: ( + keyed("presentation-v1", seeds["presentation"], f"{mode}|{scorer}|{label}"), + label, + ) + ) + presentation_rows.append(f"{mode}-{scorer}\t{','.join(labels)}") + outputs[SEALED / "presentation-orders.tsv"] = "\n".join(presentation_rows) + "\n" + + claims = [f"{mode}-{scorer}" for mode in MODES for scorer in ("s1", "s2")] + claims.sort(key=lambda claim: (keyed("scorer-v1", seeds["scorer"], claim), claim)) + outputs[SEALED / "scoring-schedule.tsv"] = "claim\n" + "\n".join(claims) + "\n" + + commitments = { + "schema_version": 1, + "algorithm": "sha256(tag_utf8 || NUL || seed_bytes)", + "commitments": { + name: sha256(f"{name}-v1".encode() + b"\0" + bytes.fromhex(seed)) + for name, seed in sorted(seeds.items()) + }, + } + outputs[FREEZE / "randomization" / "commitments.json"] = json.dumps(commitments, indent=2, sort_keys=True) + "\n" + return outputs + + +def validate_allowlists() -> None: + for mode in MODES: + path = FREEZE / "allowlists" / f"{mode}.txt" + lines = path.read_text().splitlines() + if not lines or len(lines) != len(set(lines)): + raise ValueError(f"{mode}: empty or duplicate allowlist") + if any(not re.fullmatch(r"https://doc\.rust-lang\.org/\S+", line) for line in lines): + raise ValueError(f"{mode}: allowlist is not URL-only") + filename, start_marker, end_marker = AUTHORITY_RANGES[mode] + source = (FREEZE / "oracle" / filename).read_text() + start = source.index(start_marker) + end = source.index(end_marker, start + len(start_marker)) if end_marker else len(source) + oracle_urls = re.findall(r"https://doc\.rust-lang\.org/[^`)\s]+", source[start:end]) + if lines != oracle_urls: + raise ValueError(f"{mode}: allowlist differs from canonical oracle URL order") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--write", action="store_true", help="write generated artifacts") + args = parser.parse_args() + + if args.write and (FREEZE / "LOCK.json").exists(): + raise SystemExit("refusing to rewrite generated artifacts after freeze lock") + + validate_allowlists() + outputs = generated_files(read_seeds()) + mismatches: list[str] = [] + for path, expected in outputs.items(): + if args.write: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(expected) + elif not path.exists() or path.read_text() != expected: + mismatches.append(str(path.relative_to(RUN))) + if mismatches: + raise SystemExit("generated artifact mismatch:\n" + "\n".join(mismatches)) + print(f"validated {len(outputs)} generated artifacts") + + +if __name__ == "__main__": + main() diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/protocol.py b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/protocol.py new file mode 100644 index 0000000000..f03eeb440c --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/protocol.py @@ -0,0 +1,3603 @@ +#!/usr/bin/env python3 +"""Frozen mechanics for the V3 targeted evaluation. + +This program validates immutable inputs, prepares neutral report cells, builds +blind packets, reconciles dual scores, and aggregates only after adjudication. +It never runs or builds a target. +""" + +from __future__ import annotations + +import argparse +import csv +import fcntl +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import io +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import prepare + + +RUN = Path(__file__).resolve().parent +FREEZE = RUN / "freeze" +SEALED = RUN / "sealed" +COLLECTION = RUN / "collection" +SCORING = RUN / "scoring" +RESULTS = RUN / "results" +EVALS = RUN.parent.parent +MODES = prepare.MODES +LABELS = tuple(chr(ord("A") + index) for index in range(10)) +SCORERS = ("s1", "s2") +GLOBAL_HARD_ERROR_COUNT = 12 +INFRA_FAILURE_CODES = { + "SERVICE_ERROR_BEFORE_OUTPUT", + "ORCHESTRATOR_TOOL_FAILURE", + "FILESYSTEM_FAILURE", +} +TERMINAL_REPORT_FAILURE_CODES = { + "REFUSAL", + "TIMEOUT_AFTER_WORK", + "INVALID_OUTPUT", + "SEMANTIC_NONCOMPLETION", +} +FILE_MANIFEST = FREEZE / "file-manifest.sha256" +LOCK = FREEZE / "LOCK.json" +OPERATION_LOCK = RUN / "operations.lock" +EVENT_PHASES = {"freeze", "collection", "scoring", "adjudication", "unblinding", "result"} +_AUTHENTICATED_FILE_DIGESTS: dict[str, str] | None = None + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be a positive integer") + return parsed + + +def acquire_operation_lock() -> Any: + OPERATION_LOCK.parent.mkdir(parents=True, exist_ok=True) + handle = OPERATION_LOCK.open("a+", encoding="utf-8") + fcntl.flock(handle, fcntl.LOCK_EX) + return handle + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + return sha256_bytes(path.read_bytes()) + + +def json_dump(value: Any) -> str: + return json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def is_nonblank_string(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def read_tsv(path: Path) -> list[dict[str, str]]: + with path.open(newline="") as file: + return list(csv.DictReader(file, dialect="excel-tab")) + + +def read_frozen_tsv(path: Path) -> list[dict[str, str]]: + data = read_frozen_bytes(path) + return list(csv.DictReader(io.StringIO(data.decode()), dialect="excel-tab")) + + +def read_frozen_bytes(path: Path) -> bytes: + data = path.read_bytes() + if FILE_MANIFEST.exists() and sha256_bytes(data) != frozen_file_digest(path): + raise ValueError(f"frozen input changed while being read: {path.relative_to(RUN)}") + return data + + +def read_frozen_text(path: Path) -> str: + return read_frozen_bytes(path).decode() + + +def write_once(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8", newline="") as file: + file.write(content) + file.flush() + os.fsync(file.fileno()) + + +def write_bytes_once(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as file: + file.write(content) + file.flush() + os.fsync(file.fileno()) + + +def load_schedule() -> dict[str, dict[str, str]]: + rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + return {row["run_id"]: row for row in rows} + + +def load_condition_map() -> dict[str, dict[str, str]]: + return { + row["condition_label"]: row + for row in read_frozen_tsv(SEALED / "condition-map.tsv") + } + + +def load_target_map() -> dict[str, dict[str, str]]: + return { + row["target_label"]: row for row in read_frozen_tsv(SEALED / "target-map.tsv") + } + + +def load_blind_map() -> dict[str, dict[str, str]]: + by_mode: dict[str, dict[str, str]] = {mode: {} for mode in MODES} + for row in read_frozen_tsv(SEALED / "blind-map.tsv"): + by_mode[row["mode"]][row["label"]] = row["run_id"] + return by_mode + + +def load_frozen_seeds() -> dict[str, str]: + data = json.loads(read_frozen_text(SEALED / "seeds.json")) + expected = {"condition", "schedule", "blind", "presentation", "scorer"} + if not isinstance(data, dict) or set(data) != expected: + raise ValueError("unexpected frozen seed keys") + if any( + not isinstance(value, str) + or not re.fullmatch(r"[0-9a-f]{64}", value) + or value == "0" * 64 + for value in data.values() + ): + raise ValueError("invalid frozen randomization seed") + if len(set(data.values())) != len(data): + raise ValueError("frozen randomization seeds are not distinct") + return data + + +def mode_for_run(run_id: str) -> str: + row = load_schedule()[run_id] + return load_target_map()[row["target_label"]]["mode"] + + +def atom_ids(mode: str) -> tuple[str, ...]: + text = read_frozen_text(FREEZE / "rubrics" / f"{mode}.md") + atoms = tuple(re.findall(rf"^- \*\*({mode}[1-9][0-9]*)\b", text, flags=re.MULTILINE)) + if len(atoms) != prepare.ATOM_COUNTS[mode] or len(set(atoms)) != len(atoms): + raise ValueError(f"invalid atom inventory for mode {mode}: {atoms}") + return atoms + + +def mode_hard_error_ids(mode: str) -> tuple[str, ...]: + text = read_frozen_text(FREEZE / "rubrics" / f"{mode}.md") + match = re.search( + r"^#{2,6}[^\n]*hard[^\n]*\n(.*?)(?=^#{1,6} |\Z)", + text, + flags=re.MULTILINE | re.DOTALL | re.IGNORECASE, + ) + if not match: + raise ValueError(f"missing hard-error section for mode {mode}") + body = match.group(1) + count = len(re.findall(r"^- ", body, flags=re.MULTILINE)) + if not count: + raise ValueError(f"empty hard-error section for mode {mode}") + return tuple(f"{mode}H{index}" for index in range(1, count + 1)) + + +def hard_error_ids(mode: str) -> tuple[str, ...]: + return tuple(f"G{index}" for index in range(1, GLOBAL_HARD_ERROR_COUNT + 1)) + mode_hard_error_ids(mode) + + +def tar_tree_digest(path: Path) -> str: + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"not a directory: {path}") + if any(item.is_symlink() for item in path.rglob("*")): + raise ValueError(f"symlink prohibited in frozen tree: {path}") + command = [ + "tar", + "--sort=name", + "--mtime=@0", + "--owner=0", + "--group=0", + "--numeric-owner", + "-C", + str(path), + "-cf", + "-", + ".", + ] + completed = subprocess.run(command, check=True, stdout=subprocess.PIPE) + return sha256_bytes(completed.stdout) + + +def byte_tree_digest(path: Path) -> str: + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"byte-tree root is not a real directory: {path}") + records: list[bytes] = [] + for item in sorted(path.rglob("*"), key=lambda value: value.relative_to(path).as_posix()): + relative = item.relative_to(path).as_posix() + if item.is_symlink() or not (item.is_dir() or item.is_file()): + raise ValueError(f"unsupported runtime entry: {item}") + if item.is_dir(): + records.append(f"d\0{relative}\n".encode()) + else: + data = item.read_bytes() + records.append( + f"f\0{relative}\0{len(data)}\0{sha256_bytes(data)}\n".encode() + ) + return sha256_bytes(b"".join(records)) + + +def lock_input_paths() -> list[Path]: + paths: list[Path] = [] + for name in ("prepare.py", "fetch_authority.py", "protocol.py"): + paths.append(RUN / name) + for root in (FREEZE, SEALED): + for path in root.rglob("*"): + if not path.is_file(): + continue + if path in {FILE_MANIFEST, LOCK} or "__pycache__" in path.parts: + continue + paths.append(path) + paths.sort(key=lambda path: path.relative_to(RUN).as_posix()) + if any(path.is_symlink() for path in paths): + raise ValueError("symlinks are prohibited in lock inputs") + return paths + + +def render_file_manifest() -> str: + return "".join( + f"{sha256_file(path)} {path.relative_to(RUN).as_posix()}\n" + for path in lock_input_paths() + ) + + +def verify_file_manifest() -> None: + if not FILE_MANIFEST.exists(): + raise ValueError("missing file-manifest.sha256") + expected = render_file_manifest() + actual = FILE_MANIFEST.read_text() + if actual != expected: + raise ValueError("freeze file manifest does not match current inputs") + + +def parse_file_manifest(data: bytes) -> dict[str, str]: + entries: dict[str, str] = {} + for line in data.decode().splitlines(): + digest, separator, name = line.partition(" ") + if ( + not separator + or not name + or name in entries + or not re.fullmatch(r"[0-9a-f]{64}", digest) + ): + raise ValueError("invalid frozen file-manifest row") + entries[name] = digest + return entries + + +def frozen_file_digest(path: Path) -> str: + relative = path.relative_to(RUN).as_posix() + if _AUTHENTICATED_FILE_DIGESTS is not None: + entries = _AUTHENTICATED_FILE_DIGESTS + else: + data = FILE_MANIFEST.read_bytes() + if LOCK.exists(): + lock = json.loads(LOCK.read_text()) + if lock.get("file_manifest_sha256") != sha256_bytes(data): + raise ValueError("file manifest is not authenticated by LOCK.json") + entries = parse_file_manifest(data) + if relative not in entries: + raise ValueError(f"path is not pinned by the freeze manifest: {relative}") + return entries[relative] + + +def validate_static(require_lock: bool, *, announce: bool = True) -> None: + global _AUTHENTICATED_FILE_DIGESTS + prepare.validate_allowlists() + generated = prepare.generated_files(prepare.read_seeds()) + for path, expected in generated.items(): + if not path.exists() or read_frozen_text(path) != expected: + raise ValueError(f"generated artifact mismatch: {path.relative_to(RUN)}") + + for role, package in prepare.PACKAGES.items(): + path = EVALS / package["path"] + actual = tar_tree_digest(path) + if actual != package["tree"]: + raise ValueError(f"{role} package digest mismatch: {actual}") + skill = sha256_file(path / "SKILL.md") + if skill != package["skill"]: + raise ValueError(f"{role} SKILL.md digest mismatch: {skill}") + for mode, (directory, expected) in prepare.TARGETS.items(): + actual = tar_tree_digest(EVALS / "fixtures" / "v3-targeted" / directory) + if actual != expected: + raise ValueError(f"{mode} target digest mismatch: {actual}") + + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + if len(schedule_rows) != 80: + raise ValueError(f"expected 80 schedule rows, found {len(schedule_rows)}") + if len({row["run_id"] for row in schedule_rows}) != 80: + raise ValueError("duplicate run ID") + if len({row["cell_id"] for row in schedule_rows}) != 80: + raise ValueError("duplicate cell ID") + target_map = load_target_map() + condition_map = load_condition_map() + wave_counts: Counter[tuple[str, str, str]] = Counter() + for row in schedule_rows: + mode = target_map[row["target_label"]]["mode"] + role = condition_map[row["condition_label"]]["role"] + wave_counts[(row["wave"], mode, role)] += 1 + expected_wave_counts = Counter( + (str(wave), mode, role) + for wave in range(1, 6) + for mode in MODES + for role in prepare.CONDITIONS + ) + if wave_counts != expected_wave_counts: + raise ValueError("schedule is not five complete balanced waves") + + blind = load_blind_map() + mapped_runs: list[str] = [] + for mode in MODES: + if tuple(sorted(blind[mode])) != LABELS: + raise ValueError(f"{mode} blind labels are incomplete") + for run_id in blind[mode].values(): + if mode_for_run(run_id) != mode: + raise ValueError(f"blind map crosses modes: {mode} {run_id}") + mapped_runs.append(run_id) + atom_ids(mode) + mode_hard_error_ids(mode) + if Counter(mapped_runs) != Counter(row["run_id"] for row in schedule_rows): + raise ValueError("blind map is not a bijection over scheduled runs") + + presentations = read_frozen_tsv(SEALED / "presentation-orders.tsv") + if {row["claim"] for row in presentations} != { + f"{mode}-{scorer}" for mode in MODES for scorer in SCORERS + }: + raise ValueError("presentation claims are incomplete") + for row in presentations: + if tuple(sorted(row["labels_in_order"].split(","))) != LABELS: + raise ValueError(f"invalid presentation order: {row['claim']}") + + for schema in (FREEZE / "schemas").glob("*.json"): + json.loads(read_frozen_text(schema)) + + authority = FREEZE / "authority-manifest.tsv" + if not authority.exists(): + raise ValueError("missing authority-manifest.tsv") + authority_rows = read_frozen_tsv(authority) + allowlist_pairs = [ + (mode, url) + for mode in MODES + for url in read_frozen_text(FREEZE / "allowlists" / f"{mode}.txt").splitlines() + ] + if [(row["mode"], row["requested_url"]) for row in authority_rows] != allowlist_pairs: + raise ValueError("authority manifest does not match exact allowlist sequence") + if any( + row["status"] != "200" + or row["fragment_found"] != "true" + or not re.fullmatch(r"[0-9a-f]{64}", row["sha256"]) + for row in authority_rows + ): + raise ValueError("authority manifest has an invalid retrieval record") + validate_event_ledger() + validate_preserved_artifacts() + + if require_lock: + verify_file_manifest() + lock = json.loads(LOCK.read_text()) + if set(lock) != { + "schema_version", + "status", + "file_manifest_sha256", + "review_signoffs", + "reports_collected_before_lock", + "locked_utc", + }: + raise ValueError("freeze lock has unexpected fields") + if type(lock.get("schema_version")) is not int or lock["schema_version"] != 1 or lock.get("status") != "FROZEN": + raise ValueError("invalid freeze lock") + locked_utc = lock.get("locked_utc") + if not isinstance(locked_utc, str): + raise ValueError("invalid freeze lock timestamp") + try: + locked_time = datetime.fromisoformat(locked_utc.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("invalid freeze lock timestamp") from error + if locked_time.tzinfo is None: + raise ValueError("freeze lock timestamp lacks timezone") + root = sha256_file(FILE_MANIFEST) + if lock.get("file_manifest_sha256") != root: + raise ValueError("freeze lock root does not match file manifest") + signoffs = lock.get("review_signoffs", []) + if not isinstance(signoffs, list) or len(signoffs) < 2 or any( + not isinstance(signoff, dict) for signoff in signoffs + ): + raise ValueError("freeze lock lacks two review signoffs") + reviewer_ids = [signoff.get("reviewer_id") for signoff in signoffs] + if any( + not isinstance(reviewer_id, str) + or not reviewer_id + or reviewer_id != reviewer_id.strip() + for reviewer_id in reviewer_ids + ): + raise ValueError("freeze lock has an empty reviewer ID") + if len(set(reviewer_ids)) != len(signoffs): + raise ValueError("freeze lock reviewer IDs are not distinct") + for signoff in signoffs: + if set(signoff) != { + "reviewer_id", + "verdict", + "file_manifest_sha256", + "scope", + "reviewed_utc", + }: + raise ValueError("freeze review signoff has unexpected fields") + scope = signoff.get("scope") + reviewed_utc = signoff.get("reviewed_utc") + if ( + signoff.get("verdict") != "PASS/FREEZE" + or signoff.get("file_manifest_sha256") != root + or not isinstance(scope, str) + or not scope.strip() + or scope != scope.strip() + or not isinstance(reviewed_utc, str) + or not reviewed_utc.strip() + or reviewed_utc != reviewed_utc.strip() + ): + raise ValueError("invalid freeze review signoff") + try: + reviewed_time = datetime.fromisoformat(reviewed_utc.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("invalid freeze review timestamp") from error + if reviewed_time.tzinfo is None: + raise ValueError("freeze review timestamp lacks timezone") + if ( + type(lock.get("reports_collected_before_lock")) is not int + or lock["reports_collected_before_lock"] != 0 + ): + raise ValueError("freeze lock does not attest zero prior reports") + freeze_events = [event for event in event_records() if event["event"] == "freeze_locked"] + if len(freeze_events) > 1: + raise ValueError("multiple freeze-lock events") + if freeze_events: + freeze_sequence = freeze_events[0]["sequence"] + if any( + event["sequence"] < freeze_sequence and event["phase"] != "freeze" + for event in event_records() + ): + raise ValueError("evaluation activity predates the freeze-lock event") + elif any( + path.is_file() + for root in (COLLECTION, SCORING, RESULTS) + if root.exists() + for path in root.rglob("*") + ): + raise ValueError("evaluation artifacts exist before the freeze-lock event") + _AUTHENTICATED_FILE_DIGESTS = parse_file_manifest(FILE_MANIFEST.read_bytes()) + if "**Preregistration status:** FROZEN." not in read_frozen_text( + FREEZE / "plan.md" + ): + raise ValueError("plan is not marked FROZEN") + if announce: + print("static protocol validation passed") + + +def make_read_only(path: Path) -> None: + for item in sorted(path.rglob("*"), reverse=True): + if item.is_file(): + os.utime(item, (0, 0), follow_symlinks=False) + item.chmod(0o444) + elif item.is_dir(): + os.utime(item, (0, 0), follow_symlinks=False) + item.chmod(0o555) + os.utime(path, (0, 0), follow_symlinks=False) + path.chmod(0o555) + + +def resolve_run(run_id: str) -> tuple[dict[str, str], dict[str, str], dict[str, str]]: + schedule = load_schedule() + if run_id not in schedule: + raise ValueError(f"unknown run ID: {run_id}") + row = schedule[run_id] + target = load_target_map()[row["target_label"]] + condition = load_condition_map()[row["condition_label"]] + return row, target, condition + + +def event_records() -> list[dict[str, Any]]: + path = RUN / "events.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +def assert_authority_verification_allowed(wave: int) -> None: + if wave not in range(1, 6): + raise ValueError(f"invalid collection wave: {wave}") + events = event_records() + if any( + event["event"] == "authority_verified" + and event.get("details", {}).get("wave") == wave + for event in events + ): + raise ValueError(f"wave {wave} authority verification was already recorded") + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + completed = {event.get("run_id") for event in events if event["event"] == "report_preserved"} + required_completed = { + row["run_id"] for row in schedule_rows if int(row["wave"]) < wave + } + if not required_completed <= completed: + raise ValueError(f"earlier collection waves are incomplete before wave {wave}") + prepared = {event.get("run_id") for event in events if event["event"] == "cell_prepared"} + current = {row["run_id"] for row in schedule_rows if int(row["wave"]) == wave} + if prepared & current: + raise ValueError(f"wave {wave} authority verification must precede cell preparation") + + +def assert_prepare_allowed(run_id: str) -> None: + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + order = [row["run_id"] for row in schedule_rows] + position = order.index(run_id) + row = schedule_rows[position] + events = event_records() + authority_events = [ + event + for event in events + if event["event"] == "authority_verified" + and event.get("details", {}).get("wave") == int(row["wave"]) + and event.get("sha256") == sha256_file(FREEZE / "authority-manifest.tsv") + ] + if len(authority_events) != 1: + raise ValueError(f"wave {row['wave']} lacks one current authority verification") + prepared = {event.get("run_id") for event in events if event["event"] == "cell_prepared"} + completed = {event.get("run_id") for event in events if event["event"] == "report_preserved"} + if run_id in prepared: + raise ValueError(f"cell already prepared: {run_id}") + earlier_in_wave = { + prior["run_id"] + for prior in schedule_rows[:position] + if prior["wave"] == row["wave"] + } + if not earlier_in_wave <= prepared: + raise ValueError(f"prepare order violation before {run_id}") + earlier_waves = { + prior["run_id"] for prior in schedule_rows if int(prior["wave"]) < int(row["wave"]) + } + if not earlier_waves <= completed: + raise ValueError(f"wave barrier violation before {run_id}") + if len(prepared - completed) >= 3: + raise ValueError(f"three report cells are already active before {run_id}") + + +def record_agent_start(run_id: str, attempt: int, agent_id: str) -> None: + if attempt < 1: + raise ValueError("report attempt must be positive") + if not agent_id: + raise ValueError("agent ID must be nonempty") + events = event_records() + if not any(event["event"] == "cell_prepared" and event.get("run_id") == run_id for event in events): + raise ValueError(f"cell was not prepared: {run_id}") + key = (run_id, attempt) + started = { + (event.get("run_id"), event.get("attempt")) + for event in events + if event["event"] == "agent_started" + } + returned = { + (event.get("run_id"), event.get("attempt")) + for event in events + if event["event"] == "agent_returned" + } + if key in started: + raise ValueError(f"agent attempt already started: {run_id}/{attempt}") + used_agent_ids = { + event.get("agent_id") + for event in events + if event["event"] in {"agent_started", "evaluator_started"} + } + if agent_id in used_agent_ids: + raise ValueError(f"agent ID was already used by an evaluated agent: {agent_id}") + if len(started - returned) >= 3: + raise ValueError("three report agents are already active") + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + position = next(index for index, row in enumerate(schedule_rows) if row["run_id"] == run_id) + current_wave = schedule_rows[position]["wave"] + earlier_in_wave = [ + row["run_id"] for row in schedule_rows[:position] if row["wave"] == current_wave + ] + started_runs = {item[0] for item in started} + if not all(prior in started_runs for prior in earlier_in_wave): + raise ValueError(f"agent launch-order violation before {run_id}") + if attempt > 1 and not any( + event["event"] == "infrastructure_failure" + and event.get("run_id") == run_id + and event.get("attempt") == attempt - 1 + for event in events + ): + raise ValueError("fresh attempt lacks a preceding infrastructure failure") + setup = json.loads((COLLECTION / "setups" / f"{run_id}.json").read_text()) + runtime = Path(setup["runtime_root"]) + verify_runtime(run_id, runtime) + prompt_digest = sha256_bytes(render_report_prompt(run_id, runtime).encode()) + if setup.get("report_prompt_sha256") != prompt_digest: + raise ValueError(f"rendered report prompt changed for {run_id}") + output_entries_now = list((runtime / "output").iterdir()) + if output_entries_now: + raise ValueError(f"report output is not initially empty: {run_id}/{attempt}") + append_event( + "collection", + "agent_started", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={ + "model": "gpt-5.6-sol", + "reasoning_effort": "ultra", + "fork_turns": "none", + "prompt_sha256": prompt_digest, + }, + ) + + +def record_prelaunch_failure(run_id: str, evidence: str) -> None: + if not evidence.strip(): + raise ValueError("prelaunch failure requires nonempty evidence") + resolve_run(run_id) + events = event_records() + if not any( + event["event"] == "cell_prepared" and event.get("run_id") == run_id + for event in events + ): + raise ValueError(f"cell was not prepared: {run_id}") + started = [ + event + for event in events + if event["event"] == "agent_started" and event.get("run_id") == run_id + ] + returned_attempts = { + event.get("attempt") + for event in events + if event["event"] == "agent_returned" and event.get("run_id") == run_id + } + if any(event["attempt"] not in returned_attempts for event in started): + raise ValueError("cannot record a prelaunch failure while a report agent is active") + if any( + event["event"] == "report_preserved" and event.get("run_id") == run_id + for event in events + ): + raise ValueError("cannot record a prelaunch failure after canonical completion") + next_attempt = max((event["attempt"] for event in started), default=0) + 1 + append_event( + "collection", + "prelaunch_failure", + run_id=run_id, + details={ + "disposition": "API_NO_AGENT_START", + "next_attempt": next_attempt, + "evidence": evidence, + }, + ) + + +def assert_agent_started(run_id: str, attempt: int, agent_id: str) -> None: + matches = [ + event + for event in event_records() + if event["event"] == "agent_started" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + and event.get("agent_id") == agent_id + ] + if len(matches) != 1: + raise ValueError(f"missing unique agent-start record: {run_id}/{attempt}/{agent_id}") + + +def record_reminder(run_id: str, attempt: int, agent_id: str) -> None: + assert_agent_started(run_id, attempt, agent_id) + events = event_records() + if any( + event["event"] == "reminder_sent" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + for event in events + ): + raise ValueError("the one permitted reminder was already recorded") + if any( + event["event"] == "agent_returned" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + for event in events + ): + raise ValueError("a reminder cannot follow agent return") + start = next( + event + for event in events + if event["event"] == "agent_started" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + ) + started = datetime.fromisoformat(start["time_utc"].replace("Z", "+00:00")) + if (datetime.now(timezone.utc) - started).total_seconds() < 180: + raise ValueError("reminder is not permitted before 180 seconds") + append_event( + "collection", + "reminder_sent", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"text_sha256": sha256_bytes(report_reminder_text().encode())}, + ) + + +def assert_evaluator_attempt_allowed( + kind: str, identity: str, attempt: int, events: list[dict[str, Any]] | None = None +) -> None: + if kind not in {"scorer", "adjudicator"} or attempt < 1: + raise ValueError(f"invalid evaluator attempt: {kind}/{identity}/{attempt}") + events = event_records() if events is None else events + if any( + event["event"] == "evaluator_started" + and event.get("attempt") == attempt + and event.get("details", {}).get("kind") == kind + and event.get("details", {}).get("identity") == identity + for event in events + ): + raise ValueError(f"evaluator attempt already started: {kind}/{identity}/{attempt}") + completion_event = "score_preserved" if kind == "scorer" else "adjudication_preserved" + if any( + event["event"] == completion_event + and ( + (kind == "scorer" and f"{event.get('details', {}).get('mode')}-{event.get('details', {}).get('scorer')}" == identity) + or (kind == "adjudicator" and event.get("details", {}).get("mode") == identity) + ) + for event in events + ): + raise ValueError(f"evaluator identity already completed: {kind}/{identity}") + if attempt > 1 and not any( + event["event"] == "evaluator_infrastructure_failure" + and event.get("attempt") == attempt - 1 + and event.get("details", {}).get("kind") == kind + and event.get("details", {}).get("identity") == identity + for event in events + ): + raise ValueError("fresh evaluator retry lacks a preceding infrastructure failure") + + +def record_evaluator_start(kind: str, identity: str, attempt: int, agent_id: str) -> None: + if kind not in {"scorer", "adjudicator"}: + raise ValueError(f"unknown evaluator kind: {kind}") + events = event_records() + assert_evaluator_attempt_allowed(kind, identity, attempt, events) + key = (kind, identity, attempt) + started = { + ( + event.get("details", {}).get("kind"), + event.get("details", {}).get("identity"), + event.get("attempt"), + ) + for event in events + if event["event"] == "evaluator_started" + } + returned = { + ( + event.get("details", {}).get("kind"), + event.get("details", {}).get("identity"), + event.get("attempt"), + ) + for event in events + if event["event"] == "evaluator_returned" + } + used_agent_ids = { + event.get("agent_id") + for event in events + if event["event"] in {"agent_started", "evaluator_started"} + } + if agent_id in used_agent_ids: + raise ValueError(f"agent ID was already used by an evaluated agent: {agent_id}") + if len(started - returned) >= 3: + raise ValueError("three evaluator agents are already active") + if kind == "scorer": + claims = [row["claim"] for row in read_frozen_tsv(SEALED / "scoring-schedule.tsv")] + if identity not in claims: + raise ValueError(f"unknown scorer claim: {identity}") + prior = claims[: claims.index(identity)] + started_identities = {(item[0], item[1]) for item in started} + if not all(("scorer", claim) in started_identities for claim in prior): + raise ValueError(f"scorer launch-order violation before {identity}") + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + verify_score_packet(mode, scorer) + else: + source_packet = SCORING / "adjudication-packets" / identity + if not source_packet.exists(): + raise ValueError(f"adjudication packet does not exist: {identity}") + verify_adjudication_packet(identity) + output = expected_evaluator_output(kind, identity, attempt) + verify_evaluator_runtime(kind, identity, attempt, source_packet, output) + if output.is_symlink() or not output.is_dir() or any(output.iterdir()): + raise ValueError(f"evaluator output is not initially empty: {kind}/{identity}") + append_event( + "scoring" if kind == "scorer" else "adjudication", + "evaluator_started", + attempt=attempt, + agent_id=agent_id, + details={ + "kind": kind, + "identity": identity, + "model": "gpt-5.6-sol", + "reasoning_effort": "ultra", + "fork_turns": "none", + "prompt_sha256": sha256_bytes( + render_packet_prompt( + "scorer.md" if kind == "scorer" else "adjudicator.md", + expected_evaluator_packet(kind, identity, attempt), + output, + **({"SCORER_ID": identity.split("-", 1)[1]} if kind == "scorer" else {}), + ).encode() + ), + }, + ) + + +def assert_evaluator_started( + kind: str, identity: str, attempt: int, agent_id: str +) -> None: + matches = [ + event + for event in event_records() + if event["event"] == "evaluator_started" + and event.get("attempt") == attempt + and event.get("agent_id") == agent_id + and event.get("details", {}).get("kind") == kind + and event.get("details", {}).get("identity") == identity + ] + if len(matches) != 1: + raise ValueError( + f"missing evaluator-start record: {kind}/{identity}/{attempt}/{agent_id}" + ) + + +def record_evaluator_prelaunch_failure( + kind: str, identity: str, attempt: int, output: Path, evidence: str +) -> None: + if not evidence.strip(): + raise ValueError("evaluator prelaunch failure requires nonempty evidence") + assert_evaluator_attempt_allowed(kind, identity, attempt) + if kind == "scorer": + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + verify_score_packet(mode, scorer) + else: + source_packet = SCORING / "adjudication-packets" / identity + verify_adjudication_packet(identity) + verify_evaluator_runtime(kind, identity, attempt, source_packet, output) + if any(output.iterdir()): + raise ValueError("prelaunch evaluator output is not empty") + append_event( + "scoring" if kind == "scorer" else "adjudication", + "evaluator_prelaunch_failure", + attempt=attempt, + details={ + "kind": kind, + "identity": identity, + "disposition": "API_NO_AGENT_START", + "evidence": evidence, + }, + ) + + +def prepare_cell(run_id: str, runtime: Path) -> None: + validate_static(require_lock=True, announce=False) + assert_prepare_allowed(run_id) + if runtime.exists(): + raise FileExistsError(f"runtime already exists: {runtime}") + row, target, condition = resolve_run(run_id) + expected_runtime = Path("/tmp/ur-eval") / row["cell_id"] + if runtime != expected_runtime: + raise ValueError(f"runtime must be the frozen neutral path {expected_runtime}") + package_source = EVALS / condition["package_path"] + target_source = EVALS / target["source_path"] + runtime.mkdir(parents=True) + shutil.copytree(package_source, runtime / "package") + shutil.copytree(target_source, runtime / "target") + shutil.copy2(FREEZE / "allowlists" / f"{target['mode']}.txt", runtime / "allowlist.txt") + (runtime / "output").mkdir() + if tar_tree_digest(runtime / "package") != condition["tree_sha256"]: + raise ValueError(f"package runtime copy does not match frozen identity for {run_id}") + if tar_tree_digest(runtime / "target") != target["tree_sha256"]: + raise ValueError(f"target runtime copy does not match frozen identity for {run_id}") + package_bytes = byte_tree_digest(runtime / "package") + target_bytes = byte_tree_digest(runtime / "target") + if package_bytes != byte_tree_digest(package_source): + raise ValueError(f"package runtime copy differs for {run_id}") + if target_bytes != byte_tree_digest(target_source): + raise ValueError(f"target runtime copy differs for {run_id}") + expected_allowlist = frozen_file_digest( + FREEZE / "allowlists" / f"{target['mode']}.txt" + ) + if sha256_file(runtime / "allowlist.txt") != expected_allowlist: + raise ValueError(f"allowlist runtime copy differs from frozen bytes for {run_id}") + attestation = { + "schema_version": 1, + "run_id": run_id, + "cell_id": row["cell_id"], + "runtime_root": str(runtime), + "package_byte_tree_sha256": package_bytes, + "target_byte_tree_sha256": target_bytes, + "allowlist_sha256": expected_allowlist, + "report_prompt_sha256": sha256_bytes(render_report_prompt(run_id, runtime).encode()), + "output_initially_empty": True, + "prepared_utc": utc_now(), + } + setup_path = COLLECTION / "setups" / f"{run_id}.json" + write_once(setup_path, json_dump(attestation)) + setup_path.chmod(0o444) + make_read_only(runtime / "package") + make_read_only(runtime / "target") + os.utime(runtime / "allowlist.txt", (0, 0), follow_symlinks=False) + (runtime / "allowlist.txt").chmod(0o444) + os.utime(runtime, (0, 0), follow_symlinks=False) + runtime.chmod(0o555) + append_event( + "collection", + "cell_prepared", + run_id=run_id, + digest=sha256_file(setup_path), + details={"cell_id": row["cell_id"]}, + ) + print(runtime) + + +def report_prompt_blocks() -> list[str]: + template = read_frozen_text(FREEZE / "prompts" / "report.md") + blocks = re.findall(r"```text\n(.*?)\n```", template, flags=re.DOTALL) + if len(blocks) != 2: + raise ValueError("report prompt template must contain prompt and reminder fences") + return blocks + + +def report_reminder_text() -> str: + return report_prompt_blocks()[1] + + +def render_report_prompt(run_id: str, runtime: Path) -> str: + _row, target, _condition = resolve_run(run_id) + prompt = report_prompt_blocks()[0] + replacements = { + "[PACKAGE]": str(runtime / "package"), + "[TARGET]": str(runtime / "target"), + "[URL_ALLOWLIST]": str(runtime / "allowlist.txt"), + "[OUTPUT]": str(runtime / "output"), + "[WORD_LIMIT]": target["word_cap"], + } + for old, new in replacements.items(): + prompt = prompt.replace(old, new) + if re.search(r"\[[A-Z_]+\]", prompt): + raise ValueError("unresolved report-prompt placeholder") + return prompt + + +def verify_runtime(run_id: str, runtime: Path, *, allow_invalid_output: bool = False) -> None: + setup_path = COLLECTION / "setups" / f"{run_id}.json" + prepared = [ + event + for event in event_records() + if event["event"] == "cell_prepared" and event.get("run_id") == run_id + ] + if len(prepared) != 1 or prepared[0].get("sha256") != sha256_file(setup_path): + raise ValueError(f"setup attestation changed for {run_id}") + setup = json.loads(setup_path.read_text()) + row, _target, _condition = resolve_run(run_id) + expected_runtime = Path("/tmp/ur-eval") / row["cell_id"] + if runtime != expected_runtime or runtime != Path(setup["runtime_root"]): + raise ValueError(f"runtime is not bound to frozen cell {run_id}") + if runtime.is_symlink() or not runtime.is_dir(): + raise ValueError(f"runtime root is not a real directory: {runtime}") + if runtime.stat().st_mode & 0o222: + raise ValueError(f"runtime root is writable: {runtime}") + required_inputs = {"package", "target", "allowlist.txt"} + expected_entries = required_inputs | {"output"} + actual_entries = {entry.name for entry in runtime.iterdir()} + if ( + (not allow_invalid_output and actual_entries != expected_entries) + or (allow_invalid_output and (not required_inputs <= actual_entries or actual_entries - expected_entries)) + ): + raise ValueError(f"runtime root inventory changed for {run_id}") + allowlist = runtime / "allowlist.txt" + output = runtime / "output" + if allowlist.is_symlink() or not stat.S_ISREG(allowlist.lstat().st_mode): + raise ValueError(f"runtime allowlist is not a real file: {run_id}") + if not allow_invalid_output and ( + not output.exists() + or output.is_symlink() + or not stat.S_ISDIR(output.lstat().st_mode) + ): + raise ValueError(f"runtime output is not a real directory: {run_id}") + checks = { + "package_byte_tree_sha256": byte_tree_digest(runtime / "package"), + "target_byte_tree_sha256": byte_tree_digest(runtime / "target"), + "allowlist_sha256": sha256_file(allowlist), + "report_prompt_sha256": sha256_bytes(render_report_prompt(run_id, runtime).encode()), + } + for field, actual in checks.items(): + if setup[field] != actual: + raise ValueError(f"runtime input changed for {run_id}: {field}") + + +def observe_path(path: Path) -> dict[str, Any]: + observation: dict[str, Any] = {"path": str(path)} + try: + mode = path.lstat().st_mode + except FileNotFoundError: + observation["type"] = "missing" + return observation + observation["mode"] = stat.S_IMODE(mode) + if stat.S_ISLNK(mode): + observation.update({"type": "symlink", "target": os.readlink(path)}) + elif stat.S_ISREG(mode): + data = path.read_bytes() + observation.update( + {"type": "file", "bytes": len(data), "sha256": sha256_bytes(data)} + ) + elif stat.S_ISDIR(mode): + observation["type"] = "directory" + observation["entries"] = sorted(entry.name for entry in path.iterdir()) + try: + observation["byte_tree_sha256"] = byte_tree_digest(path) + except (OSError, ValueError) as error: + observation["byte_tree_error"] = f"{type(error).__name__}: {error}" + else: + try: + observation["tar_tree_sha256"] = tar_tree_digest(path) + except (OSError, subprocess.SubprocessError, ValueError) as error: + observation["tar_tree_error"] = f"{type(error).__name__}: {error}" + else: + observation.update({"type": "special", "file_type": stat.S_IFMT(mode)}) + return observation + + +def report_runtime_forensics(run_id: str, runtime: Path, error: Exception) -> dict[str, Any]: + row, target, condition = resolve_run(run_id) + expected_runtime = Path("/tmp/ur-eval") / row["cell_id"] + if runtime != expected_runtime: + raise ValueError(f"forensic runtime is not the frozen neutral path {expected_runtime}") + setup_path = COLLECTION / "setups" / f"{run_id}.json" + prepared = [ + event + for event in event_records() + if event["event"] == "cell_prepared" and event.get("run_id") == run_id + ] + return { + "schema_version": 1, + "run_id": run_id, + "verification_error": f"{type(error).__name__}: {error}", + "expected": { + "runtime_root": str(expected_runtime), + "setup_sha256": prepared[0].get("sha256") if len(prepared) == 1 else None, + "package_tar_tree_sha256": condition["tree_sha256"], + "target_tar_tree_sha256": target["tree_sha256"], + "allowlist_sha256": frozen_file_digest( + FREEZE / "allowlists" / f"{target['mode']}.txt" + ), + }, + "observed": { + "runtime_root": observe_path(runtime), + "setup": observe_path(setup_path), + "package": observe_path(runtime / "package") + if runtime.exists() and not runtime.is_symlink() and runtime.is_dir() + else {"type": "unavailable"}, + "target": observe_path(runtime / "target") + if runtime.exists() and not runtime.is_symlink() and runtime.is_dir() + else {"type": "unavailable"}, + "allowlist": observe_path(runtime / "allowlist.txt") + if runtime.exists() and not runtime.is_symlink() and runtime.is_dir() + else {"type": "unavailable"}, + }, + } + + +def append_event( + phase: str, + event: str, + *, + run_id: str | None = None, + attempt: int | None = None, + agent_id: str | None = None, + digest: str | None = None, + details: dict[str, Any] | None = None, +) -> None: + path = RUN / "events.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+", encoding="utf-8", newline="") as file: + fcntl.flock(file, fcntl.LOCK_EX) + file.seek(0) + lines = file.read().splitlines() + sequence = len(lines) + 1 + record: dict[str, Any] = { + "schema_version": 1, + "sequence": sequence, + "previous_event_sha256": sha256_bytes(lines[-1].encode()) if lines else None, + "time_utc": utc_now(), + "phase": phase, + "event": event, + "details": details or {}, + } + if run_id is not None: + record["run_id"] = run_id + if attempt is not None: + record["attempt"] = attempt + if agent_id is not None: + record["agent_id"] = agent_id + if digest is not None: + record["sha256"] = digest + validate_event_record(record, sequence) + file.seek(0, os.SEEK_END) + file.write(json.dumps(record, sort_keys=True) + "\n") + file.flush() + os.fsync(file.fileno()) + fcntl.flock(file, fcntl.LOCK_UN) + + +def validate_event_ledger() -> None: + path = RUN / "events.jsonl" + if not path.exists(): + return + previous: str | None = None + for index, line in enumerate(path.read_text().splitlines(), start=1): + value = json.loads(line) + validate_event_record(value, index) + if value.get("previous_event_sha256") != previous: + raise ValueError(f"broken event hash chain at line {index}") + previous = sha256_bytes(line.encode()) + + +def validate_event_record(value: Any, expected_sequence: int) -> None: + required = { + "schema_version", + "sequence", + "previous_event_sha256", + "time_utc", + "phase", + "event", + "details", + } + optional = {"run_id", "attempt", "agent_id", "sha256"} + if not isinstance(value, dict) or not required <= set(value) or not set(value) <= required | optional: + raise ValueError(f"invalid event fields at sequence {expected_sequence}") + if ( + type(value["schema_version"]) is not int + or value["schema_version"] != 1 + or type(value["sequence"]) is not int + or value["sequence"] != expected_sequence + ): + raise ValueError(f"invalid event sequence {expected_sequence}") + previous = value["previous_event_sha256"] + if previous is not None and ( + not isinstance(previous, str) or not re.fullmatch(r"[0-9a-f]{64}", previous) + ): + raise ValueError(f"invalid prior-event hash at sequence {expected_sequence}") + if not isinstance(value["time_utc"], str): + raise ValueError(f"invalid event timestamp at sequence {expected_sequence}") + try: + parsed_time = datetime.fromisoformat(value["time_utc"].replace("Z", "+00:00")) + except (AttributeError, ValueError) as error: + raise ValueError(f"invalid event timestamp at sequence {expected_sequence}") from error + if parsed_time.tzinfo is None: + raise ValueError(f"event timestamp lacks timezone at sequence {expected_sequence}") + if ( + not isinstance(value["phase"], str) + or value["phase"] not in EVENT_PHASES + or not isinstance(value["event"], str) + or not value["event"] + ): + raise ValueError(f"invalid event identity at sequence {expected_sequence}") + if not isinstance(value["details"], dict): + raise ValueError(f"invalid event details at sequence {expected_sequence}") + if "run_id" in value and (not isinstance(value["run_id"], str) or not value["run_id"]): + raise ValueError(f"invalid event run ID at sequence {expected_sequence}") + if "attempt" in value and ( + not isinstance(value["attempt"], int) + or isinstance(value["attempt"], bool) + or value["attempt"] < 1 + ): + raise ValueError(f"invalid event attempt at sequence {expected_sequence}") + if "agent_id" in value and ( + not is_nonblank_string(value["agent_id"]) + or value["agent_id"] != value["agent_id"].strip() + ): + raise ValueError(f"invalid event agent ID at sequence {expected_sequence}") + if "sha256" in value and ( + not isinstance(value["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) + ): + raise ValueError(f"invalid event digest at sequence {expected_sequence}") + + +def validate_preserved_artifacts() -> None: + events = event_records() + attempt_events: dict[tuple[str, int], dict[str, Any]] = {} + evaluator_attempt_events: dict[tuple[str, str, int], dict[str, Any]] = {} + invalid_output_events: dict[tuple[str, str, int], dict[str, Any]] = {} + for event in events: + if event["event"] == "attempt_preserved": + key = (event["run_id"], event["attempt"]) + if key in attempt_events: + raise ValueError(f"duplicate attempt-preservation event: {key}") + attempt_events[key] = event + directory = ( + COLLECTION + / "attempts" + / event["run_id"] + / str(event["attempt"]) + ) + if byte_tree_digest(directory) != event.get("sha256"): + raise ValueError( + f"preserved attempt changed: {event['run_id']}/{event['attempt']}" + ) + elif event["event"] == "invalid_output_preserved": + details = event.get("details", {}) + kind = details.get("kind") + if kind not in {"scorer", "adjudicator"}: + raise ValueError("invalid preserved-output evaluator kind") + phase = "scoring" if kind == "scorer" else "adjudication" + identity = details.get("identity") + key = (phase, str(identity), event["attempt"]) + if event["phase"] != phase or key in invalid_output_events: + raise ValueError(f"duplicate or misphased invalid-output event: {key}") + invalid_output_events[key] = event + directory = ( + SCORING + / "invalid" + / phase + / str(identity) + / str(event.get("attempt")) + ) + if byte_tree_digest(directory) != event.get("sha256"): + raise ValueError(f"preserved invalid output changed: {directory}") + elif event["event"] == "evaluator_attempt_preserved": + details = event.get("details", {}) + kind = details.get("kind") + if kind not in {"scorer", "adjudicator"}: + raise ValueError("invalid preserved-attempt evaluator kind") + phase = "scoring" if kind == "scorer" else "adjudication" + identity = details.get("identity") + key = (phase, str(identity), event["attempt"]) + if event["phase"] != phase or key in evaluator_attempt_events: + raise ValueError(f"duplicate or misphased evaluator-attempt event: {key}") + evaluator_attempt_events[key] = event + directory = ( + SCORING + / "evaluator-attempts" + / phase + / str(identity) + / str(event.get("attempt")) + ) + if byte_tree_digest(directory) != event.get("sha256"): + raise ValueError(f"preserved evaluator attempt changed: {directory}") + elif event["event"] == "freeze_locked": + if ( + sha256_file(FILE_MANIFEST) != event.get("sha256") + or sha256_file(LOCK) != event.get("details", {}).get("lock_sha256") + ): + raise ValueError("freeze-lock artifacts changed") + elif event["event"] == "authority_verified": + if frozen_file_digest(FREEZE / "authority-manifest.tsv") != event.get("sha256"): + raise ValueError("authority verification event has the wrong manifest") + elif event["event"] == "report_preserved": + report = ( + COLLECTION + / "attempts" + / event["run_id"] + / str(event["attempt"]) + / "report.md" + ) + if sha256_file(report) != event.get("sha256"): + raise ValueError(f"preserved report changed: {event['run_id']}") + elif event["event"] == "collection_locked": + if sha256_file(COLLECTION / "valid-index.jsonl") != event.get("sha256"): + raise ValueError("locked collection index changed") + elif event["event"] == "blind_packet_preserved": + details = event.get("details", {}) + packet = SCORING / "packets" / str(details.get("mode")) / str( + details.get("scorer") + ) + if byte_tree_digest(packet) != event.get("sha256"): + raise ValueError(f"preserved blind packet changed: {packet}") + elif event["event"] == "score_preserved": + details = event.get("details", {}) + score = SCORING / "raw" / str(details.get("mode")) / ( + str(details.get("scorer")) + ".json" + ) + if sha256_file(score) != event.get("sha256"): + raise ValueError(f"preserved score changed: {score}") + elif event["event"] == "disagreements_materialized": + path = SCORING / "disagreements" / ( + str(event.get("details", {}).get("mode")) + ".json" + ) + if sha256_file(path) != event.get("sha256"): + raise ValueError(f"preserved disagreements changed: {path}") + elif event["event"] == "adjudication_packet_preserved": + packet = SCORING / "adjudication-packets" / str( + event.get("details", {}).get("mode") + ) + if byte_tree_digest(packet) != event.get("sha256"): + raise ValueError(f"preserved adjudication packet changed: {packet}") + elif event["event"] == "adjudication_preserved": + path = SCORING / "adjudications" / ( + str(event.get("details", {}).get("mode")) + ".json" + ) + if sha256_file(path) != event.get("sha256"): + raise ValueError(f"preserved adjudication changed: {path}") + elif event["event"] == "final_blind_score_locked": + path = SCORING / "final" / ( + str(event.get("details", {}).get("mode")) + ".json" + ) + if sha256_file(path) != event.get("sha256"): + raise ValueError(f"final blind score changed: {path}") + elif event["event"] == "conditions_revealed": + if sha256_file(RUN / "unblinding.json") != event.get("sha256"): + raise ValueError("unblinding artifact changed") + elif event["event"] == "aggregate_written": + if ( + sha256_file(RESULTS / "aggregate.json") != event.get("sha256") + or sha256_file(RESULTS / "summary.md") + != event.get("details", {}).get("summary_sha256") + ): + raise ValueError("final result artifacts changed") + elif event["event"] == "run_invalidated": + if sha256_file(RUN / "INVALID.json") != event.get("sha256"): + raise ValueError("INVALID marker changed") + attempt_root = COLLECTION / "attempts" + actual_attempts: set[tuple[str, int]] = set() + if attempt_root.exists(): + for run_dir in attempt_root.iterdir(): + if run_dir.is_symlink() or not run_dir.is_dir(): + raise ValueError(f"invalid attempt run directory: {run_dir}") + for attempt_dir in run_dir.iterdir(): + if ( + attempt_dir.is_symlink() + or not attempt_dir.is_dir() + or not attempt_dir.name.isdigit() + or int(attempt_dir.name) < 1 + ): + raise ValueError(f"invalid attempt directory: {attempt_dir}") + actual_attempts.add((run_dir.name, int(attempt_dir.name))) + if actual_attempts != set(attempt_events): + raise ValueError("attempt directories and preservation events differ") + + def inventory_evaluator_attempts( + root: Path, expected: set[tuple[str, str, int]], label: str + ) -> set[tuple[str, str, int]]: + if root.is_symlink(): + raise ValueError(f"invalid {label} artifact root: {root}") + if not root.exists(): + if expected: + raise ValueError(f"missing {label} artifact root") + return set() + if not root.is_dir(): + raise ValueError(f"invalid {label} artifact root: {root}") + if not expected: + raise ValueError(f"orphan empty {label} artifact root: {root}") + actual: set[tuple[str, str, int]] = set() + actual_phases: set[str] = set() + actual_identities: set[tuple[str, str]] = set() + scorer_identities = { + f"{mode}-{scorer}" for mode in MODES for scorer in SCORERS + } + for phase_dir in root.iterdir(): + if ( + phase_dir.is_symlink() + or not phase_dir.is_dir() + or phase_dir.name not in {"scoring", "adjudication"} + ): + raise ValueError(f"invalid {label} phase directory: {phase_dir}") + phase = phase_dir.name + actual_phases.add(phase) + for identity_dir in phase_dir.iterdir(): + identity = identity_dir.name + valid_identity = ( + identity in scorer_identities if phase == "scoring" else identity in MODES + ) + if identity_dir.is_symlink() or not identity_dir.is_dir() or not valid_identity: + raise ValueError(f"invalid {label} identity directory: {identity_dir}") + actual_identities.add((phase, identity)) + for attempt_dir in identity_dir.iterdir(): + if ( + attempt_dir.is_symlink() + or not attempt_dir.is_dir() + or not re.fullmatch(r"[1-9][0-9]*", attempt_dir.name) + ): + raise ValueError(f"invalid {label} attempt directory: {attempt_dir}") + actual.add((phase, identity, int(attempt_dir.name))) + expected_phases = {phase for phase, _identity, _attempt in expected} + expected_identities = {(phase, identity) for phase, identity, _attempt in expected} + if actual_phases != expected_phases or actual_identities != expected_identities: + raise ValueError(f"{label} directory hierarchy differs from preservation events") + return actual + + actual_evaluator_attempts = inventory_evaluator_attempts( + SCORING / "evaluator-attempts", + set(evaluator_attempt_events), + "evaluator-attempt", + ) + if actual_evaluator_attempts != set(evaluator_attempt_events): + raise ValueError("evaluator-attempt directories and preservation events differ") + actual_invalid_outputs = inventory_evaluator_attempts( + SCORING / "invalid", set(invalid_output_events), "invalid-output" + ) + if actual_invalid_outputs != set(invalid_output_events): + raise ValueError("invalid-output directories and preservation events differ") + if set(evaluator_attempt_events) & set(invalid_output_events): + raise ValueError("one evaluator attempt has both valid/infra and invalid preservation") + + invalidated = [event for event in events if event["event"] == "run_invalidated"] + invalid_returns = [ + event + for event in events + if event["event"] == "evaluator_returned" + and event.get("details", {}).get("api_state") == "INVALID_OUTPUT" + ] + invalid_marker = RUN / "INVALID.json" + if not invalid_output_events: + if invalid_marker.exists() or invalid_marker.is_symlink() or invalidated or invalid_returns: + raise ValueError("INVALID marker or terminal-invalid events lack an invalid output") + else: + if len(invalid_output_events) != 1 or len(invalidated) != 1 or len(invalid_returns) != 1: + raise ValueError("terminal-invalid evaluator state is not unique and complete") + if invalid_marker.is_symlink() or not invalid_marker.is_file(): + raise ValueError("terminal-invalid event lacks a real INVALID marker") + key, invalid_event = next(iter(invalid_output_events.items())) + phase, identity, attempt = key + invalid_directory = SCORING / "invalid" / phase / identity / str(attempt) + attestation = json.loads((invalid_directory / "attestation.json").read_text()) + marker = json.loads(invalid_marker.read_text()) + if marker != attestation: + raise ValueError("INVALID marker differs from the invalid-output attestation") + if ( + marker.get("phase") != phase + or marker.get("identity") != identity + or marker.get("attempt") != attempt + or marker.get("agent_id") != invalid_event.get("agent_id") + or marker.get("disposition") != "INVALID_NONRERUNNABLE_EVALUATOR_OUTPUT" + ): + raise ValueError("terminal-invalid attestation identity is inconsistent") + invalidated_event = invalidated[0] + if ( + invalidated_event.get("phase") != phase + or invalidated_event.get("attempt") != attempt + or invalidated_event.get("agent_id") != marker.get("agent_id") + or invalidated_event.get("sha256") != sha256_file(invalid_marker) + or invalidated_event.get("details") + != {"identity": identity, "evidence": marker.get("evidence")} + ): + raise ValueError("run-invalidated event differs from the INVALID marker") + returned = invalid_returns[0] + expected_kind = "scorer" if phase == "scoring" else "adjudicator" + if ( + returned.get("phase") != phase + or returned.get("attempt") != attempt + or returned.get("agent_id") != marker.get("agent_id") + or returned.get("details") + != {"kind": expected_kind, "identity": identity, "api_state": "INVALID_OUTPUT"} + ): + raise ValueError("invalid evaluator-return event is inconsistent") + + setup_root = COLLECTION / "setups" + actual_setups: set[str] = set() + if setup_root.exists(): + for path in setup_root.iterdir(): + if path.is_symlink() or not path.is_file() or path.suffix != ".json": + raise ValueError(f"invalid setup artifact: {path}") + actual_setups.add(path.stem) + prepared = { + event["run_id"]: event + for event in events + if event["event"] == "cell_prepared" + } + if len(prepared) != sum(event["event"] == "cell_prepared" for event in events): + raise ValueError("duplicate cell-preparation event") + if actual_setups != set(prepared): + raise ValueError("cell setup files and preparation events differ") + for run_id, event in prepared.items(): + if sha256_file(setup_root / f"{run_id}.json") != event.get("sha256"): + raise ValueError(f"cell setup changed: {run_id}") + + +def append_report_index(metadata: dict[str, Any]) -> None: + path = COLLECTION / "valid-index.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+", encoding="utf-8", newline="") as file: + fcntl.flock(file, fcntl.LOCK_EX) + file.seek(0) + existing = [json.loads(line) for line in file.read().splitlines()] + if any(item["run_id"] == metadata["run_id"] for item in existing): + raise ValueError(f"canonical report already indexed for {metadata['run_id']}") + file.seek(0, os.SEEK_END) + file.write(json.dumps(metadata, sort_keys=True) + "\n") + file.flush() + os.fsync(file.fileno()) + fcntl.flock(file, fcntl.LOCK_UN) + + +def output_entries(output: Path) -> list[Path]: + if not output.exists() or output.is_symlink() or not stat.S_ISDIR(output.lstat().st_mode): + raise ValueError(f"output path is not a real directory: {output}") + entries = sorted(output.iterdir(), key=lambda path: path.name) if output.exists() else [] + for entry in entries: + mode = entry.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISREG(mode): + raise ValueError(f"unsupported output entry: {entry}") + return entries + + +def snapshot_path(source: Path, raw_root: Path) -> list[dict[str, Any]]: + """Preserve every regular byte reachable without following a symlink.""" + raw_root.mkdir(parents=True, exist_ok=True) + records: list[dict[str, Any]] = [] + try: + root_mode = source.lstat().st_mode + except FileNotFoundError: + return [{"path": ".", "type": "missing"}] + if stat.S_ISLNK(root_mode): + return [{"path": ".", "type": "symlink", "target": os.readlink(source)}] + if stat.S_ISREG(root_mode): + data = source.read_bytes() + stored = "__output_path_file__" + write_bytes_once(raw_root / stored, data) + return [ + { + "path": ".", + "type": "file", + "stored_as": stored, + "bytes": len(data), + "sha256": sha256_bytes(data), + } + ] + if not stat.S_ISDIR(root_mode): + return [{"path": ".", "type": "special", "mode": stat.S_IFMT(root_mode)}] + + records.append({"path": ".", "type": "directory"}) + + def walk(directory: Path, relative: Path) -> None: + with os.scandir(directory) as iterator: + entries = sorted(iterator, key=lambda entry: entry.name) + for entry in entries: + child_relative = relative / entry.name + rendered = child_relative.as_posix() + mode = entry.stat(follow_symlinks=False).st_mode + source_child = directory / entry.name + destination_child = raw_root / child_relative + if stat.S_ISREG(mode): + data = source_child.read_bytes() + write_bytes_once(destination_child, data) + records.append( + { + "path": rendered, + "type": "file", + "bytes": len(data), + "sha256": sha256_bytes(data), + } + ) + elif stat.S_ISDIR(mode): + destination_child.mkdir() + records.append({"path": rendered, "type": "directory"}) + walk(source_child, child_relative) + elif stat.S_ISLNK(mode): + records.append( + {"path": rendered, "type": "symlink", "target": os.readlink(source_child)} + ) + else: + records.append( + {"path": rendered, "type": "special", "mode": stat.S_IFMT(mode)} + ) + + walk(source, Path()) + return records + + +def preserve_captured_file(raw_root: Path, filename: str, data: bytes) -> list[dict[str, Any]]: + """Preserve already-captured bytes as an exact one-file output snapshot.""" + if Path(filename).name != filename or filename in {"", ".", ".."}: + raise ValueError(f"invalid captured output filename: {filename}") + raw_root.mkdir(parents=True, exist_ok=False) + write_bytes_once(raw_root / filename, data) + return [ + {"path": ".", "type": "directory"}, + { + "path": filename, + "type": "file", + "bytes": len(data), + "sha256": sha256_bytes(data), + }, + ] + + +def record_report( + run_id: str, + attempt: int, + runtime: Path, + agent_id: str, + scope_deviation: bool, + scope_evidence: str, +) -> None: + if scope_deviation and not scope_evidence.strip(): + raise ValueError("scope deviation requires nonempty evidence") + assert_agent_started(run_id, attempt, agent_id) + verify_runtime(run_id, runtime) + output = runtime / "output" + if not output.exists() or output.is_symlink() or not stat.S_ISDIR(output.lstat().st_mode): + raise ValueError(f"output path is not a real directory: {output}") + entries = output_entries(output) + if [entry.name for entry in entries] != ["report.md"]: + raise ValueError(f"{run_id} output is not exactly one report.md") + data = entries[0].read_bytes() + text = data.decode("utf-8") + if not text.strip(): + raise ValueError(f"{run_id} report.md is empty or whitespace-only") + words = len(text.split()) + _row, target, _condition = resolve_run(run_id) + destination = COLLECTION / "attempts" / run_id / str(attempt) + if destination.exists(): + raise FileExistsError(f"attempt already preserved: {destination}") + destination.mkdir(parents=True) + write_bytes_once(destination / "report.md", data) + metadata = { + "schema_version": 1, + "run_id": run_id, + "attempt": attempt, + "agent_id": agent_id, + "report_sha256": sha256_bytes(data), + "word_count": words, + "word_cap": int(target["word_cap"]), + "within_word_cap": words <= int(target["word_cap"]), + "utf8": True, + "canonical_for_scoring": True, + "semantic_noncompletion": False, + "terminal_disposition": "COMPLETE", + "api_state": "COMPLETED", + "operational_scope_deviation": scope_deviation, + "scope_evidence": scope_evidence, + "source_isolation": "procedural", + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(metadata)) + append_report_index(metadata) + make_read_only(destination) + append_event( + "collection", + "attempt_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"disposition": "COMPLETE"}, + ) + append_event( + "collection", + "agent_returned", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"api_state": "COMPLETED"}, + ) + append_event( + "collection", + "report_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=metadata["report_sha256"], + details={ + "word_count": words, + "within_word_cap": metadata["within_word_cap"], + "operational_scope_deviation": scope_deviation, + "disposition": "COMPLETE", + }, + ) + print(json_dump(metadata), end="") + + +def preserve_failed_report_attempt( + run_id: str, + attempt: int, + runtime: Path, + agent_id: str, + disposition: str, + evidence: str, + scope_deviation: bool, +) -> None: + if disposition not in INFRA_FAILURE_CODES | TERMINAL_REPORT_FAILURE_CODES: + raise ValueError(f"unknown failure disposition: {disposition}") + if not evidence.strip(): + raise ValueError("failed report disposition requires nonempty evidence") + assert_agent_started(run_id, attempt, agent_id) + forensics: dict[str, Any] | None = None + try: + verify_runtime(run_id, runtime, allow_invalid_output=True) + except (OSError, ValueError) as error: + forensics = report_runtime_forensics(run_id, runtime, error) + disposition = "INVALID_OUTPUT" + scope_deviation = True + evidence = ( + f"{evidence} Runtime/input verification failed: " + f"{forensics['verification_error']}" + ) + output = runtime / "output" + destination = COLLECTION / "attempts" / run_id / str(attempt) + if destination.exists(): + raise FileExistsError(f"attempt already preserved: {destination}") + if forensics is None: + entry_manifest = snapshot_path(output, destination / "raw-output") + manifest_path = destination / "raw-output-manifest.json" + captured_report = destination / "raw-output" / "report.md" + else: + # A failed runtime check may have been caused by an unexpected root entry + # or by drift in an input tree. Preserve the entire neutral runtime without + # following symlinks, rather than retaining only the ordinary output path. + entry_manifest = snapshot_path(runtime, destination / "raw-runtime") + manifest_path = destination / "raw-runtime-manifest.json" + setup = COLLECTION / "setups" / f"{run_id}.json" + setup_manifest = snapshot_path(setup, destination / "setup-at-verification") + write_once( + destination / "setup-at-verification-manifest.json", + json_dump(setup_manifest), + ) + captured_report = destination / "raw-runtime" / "output" / "report.md" + usable_report: bytes | None = None + if ( + captured_report.exists() + and not captured_report.is_symlink() + and stat.S_ISREG(captured_report.lstat().st_mode) + ): + data = captured_report.read_bytes() + try: + data.decode("utf-8") + except UnicodeDecodeError: + pass + else: + usable_report = data + write_once(manifest_path, json_dump(entry_manifest)) + if forensics is not None: + write_once(destination / "runtime-forensics.json", json_dump(forensics)) + infrastructure = disposition in INFRA_FAILURE_CODES + if infrastructure: + metadata = { + "schema_version": 1, + "run_id": run_id, + "attempt": attempt, + "agent_id": agent_id, + "terminal_disposition": disposition, + "api_state": "INFRASTRUCTURE_FAILURE", + "rerunnable": True, + "evidence": evidence, + "operational_scope_deviation": scope_deviation, + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(metadata)) + make_read_only(destination) + append_event( + "collection", + "attempt_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"disposition": disposition}, + ) + append_event( + "collection", + "agent_returned", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"api_state": "INFRASTRUCTURE_FAILURE"}, + ) + append_event( + "collection", + "infrastructure_failure", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"disposition": disposition, "evidence": evidence}, + ) + runtime.chmod(0o755) + try: + if output.is_symlink() or output.is_file(): + output.unlink() + elif output.exists(): + shutil.rmtree(output) + output.mkdir() + finally: + os.utime(runtime, (0, 0), follow_symlinks=False) + runtime.chmod(0o555) + print(json_dump(metadata), end="") + return + + if usable_report is None: + usable_report = ( + "# Evaluator-marked failed replicate\n\n" + f"No usable canonical report was produced. Terminal disposition: {disposition}.\n" + ).encode() + text = usable_report.decode("utf-8") + words = len(text.split()) + _row, target, _condition = resolve_run(run_id) + write_bytes_once(destination / "report.md", usable_report) + metadata = { + "schema_version": 1, + "run_id": run_id, + "attempt": attempt, + "agent_id": agent_id, + "report_sha256": sha256_bytes(usable_report), + "word_count": words, + "word_cap": int(target["word_cap"]), + "within_word_cap": words <= int(target["word_cap"]), + "utf8": True, + "canonical_for_scoring": True, + "semantic_noncompletion": True, + "terminal_disposition": disposition, + "api_state": "TERMINAL_NONCOMPLETION", + "operational_scope_deviation": scope_deviation or disposition == "INVALID_OUTPUT", + "scope_evidence": evidence, + "source_isolation": "procedural", + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(metadata)) + append_report_index(metadata) + make_read_only(destination) + append_event( + "collection", + "attempt_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"disposition": disposition}, + ) + append_event( + "collection", + "agent_returned", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"api_state": "TERMINAL_NONCOMPLETION"}, + ) + append_event( + "collection", + "report_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=metadata["report_sha256"], + details={"disposition": disposition, "semantic_noncompletion": True}, + ) + print(json_dump(metadata), end="") + + +def verify_collection_lock() -> None: + path = COLLECTION / "valid-index.jsonl" + matches = [event for event in event_records() if event["event"] == "collection_locked"] + if len(matches) != 1 or matches[0].get("sha256") != sha256_file(path): + raise ValueError("canonical collection index is not locked to the event ledger") + if matches[0].get("details", {}).get("report_count") != 80: + raise ValueError("collection lock has an invalid report count") + + +def load_index(*, require_collection_lock: bool = False) -> dict[str, dict[str, Any]]: + path = COLLECTION / "valid-index.jsonl" + if path.is_symlink() or not path.is_file(): + raise ValueError("canonical report index is not a real file") + rows = [json.loads(line) for line in path.read_text().splitlines()] + if any(not isinstance(row, dict) or not isinstance(row.get("run_id"), str) for row in rows): + raise ValueError("canonical report index has an invalid row") + result = {row["run_id"]: row for row in rows} + if len(rows) != len(result): + raise ValueError("duplicate canonical reports in index") + if require_collection_lock: + verify_collection_lock() + for run_id, row in result.items(): + if ( + not isinstance(row, dict) + or type(row.get("schema_version")) is not int + or row["schema_version"] != 1 + or type(row.get("attempt")) is not int + or row["attempt"] < 1 + or not isinstance(row.get("agent_id"), str) + or not row["agent_id"] + or not isinstance(row.get("report_sha256"), str) + or not re.fullmatch(r"[0-9a-f]{64}", row["report_sha256"]) + or type(row.get("word_count")) is not int + or row["word_count"] < 0 + ): + raise ValueError(f"invalid canonical report index row for {run_id}") + attempt = row["attempt"] + directory = COLLECTION / "attempts" / run_id / str(attempt) + report = directory / "report.md" + attestation = json.loads((directory / "attestation.json").read_text()) + if attestation != row: + raise ValueError(f"indexed attestation mismatch for {run_id}") + if sha256_file(report) != row["report_sha256"]: + raise ValueError(f"indexed report hash mismatch for {run_id}") + text = report.read_bytes().decode("utf-8") + if len(text.split()) != row["word_count"]: + raise ValueError(f"indexed report word-count mismatch for {run_id}") + preserved = [ + event + for event in event_records() + if event["event"] == "report_preserved" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + ] + if len(preserved) != 1 or preserved[0].get("sha256") != row["report_sha256"]: + raise ValueError(f"canonical report lacks its preservation event: {run_id}") + attempts = [ + event + for event in event_records() + if event["event"] == "attempt_preserved" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + ] + if len(attempts) != 1 or attempts[0].get("sha256") != byte_tree_digest(directory): + raise ValueError(f"canonical attempt differs from its preservation event: {run_id}") + return result + + +def presentation_order(mode: str, scorer: str) -> list[str]: + claim = f"{mode}-{scorer}" + rows = { + row["claim"]: row + for row in read_frozen_tsv(SEALED / "presentation-orders.tsv") + } + return rows[claim]["labels_in_order"].split(",") + + +def build_score_packets() -> None: + validate_static(require_lock=True, announce=False) + index = load_index() + if set(index) != set(load_schedule()): + raise ValueError("collection does not contain exactly all 80 scheduled reports") + if any(event["event"] == "collection_locked" for event in event_records()): + raise ValueError("collection was already locked") + append_event( + "collection", + "collection_locked", + digest=sha256_file(COLLECTION / "valid-index.jsonl"), + details={"report_count": 80}, + ) + (COLLECTION / "valid-index.jsonl").chmod(0o444) + verify_collection_lock() + blind = load_blind_map() + target_by_mode = {row["mode"]: row for row in load_target_map().values()} + packet_root = SCORING / "packets" + if packet_root.exists(): + raise FileExistsError("scorer packets already exist") + packet_digests: dict[str, str] = {} + for mode in MODES: + target_source = EVALS / target_by_mode[mode]["source_path"] + for scorer in SCORERS: + packet = packet_root / mode / scorer + (packet / "reports").mkdir(parents=True) + shutil.copytree(target_source, packet / "target") + shutil.copy2(FREEZE / "allowlists" / f"{mode}.txt", packet / "allowlist.txt") + shutil.copy2(FREEZE / "rubrics" / "SCORER.md", packet / "SCORER.md") + shutil.copy2(FREEZE / "rubrics" / f"{mode}.md", packet / "RUBRIC.md") + shutil.copy2(FREEZE / "schemas" / "score.schema.json", packet / "score.schema.json") + if tar_tree_digest(packet / "target") != target_by_mode[mode]["tree_sha256"]: + raise ValueError(f"copied target lost its frozen identity: {mode}/{scorer}") + frozen_copies = { + packet / "allowlist.txt": FREEZE / "allowlists" / f"{mode}.txt", + packet / "SCORER.md": FREEZE / "rubrics" / "SCORER.md", + packet / "RUBRIC.md": FREEZE / "rubrics" / f"{mode}.md", + packet / "score.schema.json": FREEZE / "schemas" / "score.schema.json", + } + for destination, source in frozen_copies.items(): + if sha256_file(destination) != frozen_file_digest(source): + raise ValueError(f"frozen packet input changed during copy: {destination}") + report_hashes: dict[str, str] = {} + for label, run_id in blind[mode].items(): + attempt = index[run_id]["attempt"] + source = COLLECTION / "attempts" / run_id / str(attempt) / "report.md" + if sha256_file(source) != index[run_id]["report_sha256"]: + raise ValueError(f"report changed before packet build: {run_id}") + destination = packet / "reports" / f"{label}.md" + shutil.copy2(source, destination) + report_hashes[label] = sha256_file(destination) + if report_hashes[label] != index[run_id]["report_sha256"]: + raise ValueError(f"report changed during packet copy: {run_id}") + manifest = { + "schema_version": 1, + "mode": mode, + "scorer_id": scorer, + "presentation_order": presentation_order(mode, scorer), + "report_sha256": report_hashes, + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "score.schema.json"), + } + write_once(packet / "PACKET.json", json_dump(manifest)) + make_read_only(packet) + packet_digest = byte_tree_digest(packet) + packet_digests[f"{mode}-{scorer}"] = packet_digest + append_event( + "scoring", + "blind_packet_preserved", + digest=packet_digest, + details={"mode": mode, "scorer": scorer}, + ) + digest = sha256_bytes(json.dumps(packet_digests, sort_keys=True).encode()) + append_event( + "scoring", + "blind_packets_built", + digest=digest, + details={"packet_count": 16}, + ) + print("built 16 blind scorer packets") + + +def verify_score_packet(mode: str, scorer: str) -> None: + verify_collection_lock() + packet = SCORING / "packets" / mode / scorer + manifest = json.loads((packet / "PACKET.json").read_text()) + if ( + type(manifest.get("schema_version")) is not int + or manifest["schema_version"] != 1 + or manifest.get("mode") != mode + or manifest.get("scorer_id") != scorer + ): + raise ValueError(f"score packet identity mismatch: {mode}/{scorer}") + if manifest.get("presentation_order") != presentation_order(mode, scorer): + raise ValueError(f"score packet presentation mismatch: {mode}/{scorer}") + checks = { + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "score.schema.json"), + } + for field, actual in checks.items(): + if manifest.get(field) != actual: + raise ValueError(f"score packet changed: {mode}/{scorer}/{field}") + report_hashes = { + label: sha256_file(packet / "reports" / f"{label}.md") for label in LABELS + } + if manifest.get("report_sha256") != report_hashes: + raise ValueError(f"score packet reports changed: {mode}/{scorer}") + expected_files = { + "PACKET.json", + "allowlist.txt", + "RUBRIC.md", + "SCORER.md", + "score.schema.json", + *(f"reports/{label}.md" for label in LABELS), + } + expected_files.update( + path.relative_to(packet).as_posix() + for path in (packet / "target").rglob("*") + if path.is_file() + ) + actual_files = { + path.relative_to(packet).as_posix() for path in packet.rglob("*") if path.is_file() + } + if actual_files != expected_files: + raise ValueError(f"score packet has unexpected files: {mode}/{scorer}") + expected_packet_digest = preserved_digest( + "blind_packet_preserved", mode=mode, scorer=scorer + ) + if byte_tree_digest(packet) != expected_packet_digest: + raise ValueError(f"score packet differs from its external preservation event: {mode}/{scorer}") + + +def render_packet_prompt(template_name: str, packet: Path, output: Path, **values: str) -> str: + template = read_frozen_text(FREEZE / "prompts" / template_name) + match = re.search(r"```text\n(.*?)\n```", template, flags=re.DOTALL) + if not match: + raise ValueError(f"{template_name} missing text fence") + prompt = match.group(1).replace("[PACKET]", str(packet)).replace("[OUTPUT]", str(output)) + for key, value in values.items(): + prompt = prompt.replace(f"[{key}]", value) + if re.search(r"\[[A-Z_]+\]", prompt): + raise ValueError(f"unresolved placeholder in {template_name}") + return prompt + + +def expected_evaluator_output(kind: str, identity: str, attempt: int) -> Path: + if attempt < 1: + raise ValueError("evaluator attempt must be positive") + seeds = load_frozen_seeds() + seed_name = "scorer" if kind == "scorer" else "presentation" + token = prepare.keyed( + f"{kind}-output-v2", seeds[seed_name], f"{identity}|{attempt}" + )[:32] + return Path("/tmp/ur-eval") / token / "output" + + +def expected_evaluator_packet(kind: str, identity: str, attempt: int) -> Path: + return expected_evaluator_output(kind, identity, attempt).parent / "packet" + + +def expected_source_packet_digest(kind: str, identity: str) -> str: + if kind == "scorer": + mode, scorer = identity.split("-", 1) + return preserved_digest("blind_packet_preserved", mode=mode, scorer=scorer) + return preserved_digest("adjudication_packet_preserved", mode=identity) + + +def prepare_evaluator_runtime( + kind: str, identity: str, attempt: int, source_packet: Path, output: Path +) -> Path: + assert_evaluator_attempt_allowed(kind, identity, attempt) + expected = expected_evaluator_output(kind, identity, attempt) + if output != expected: + raise ValueError(f"{kind} output must be the frozen neutral path {expected}") + root = output.parent + if root.exists(): + verify_evaluator_runtime(kind, identity, attempt, source_packet, output) + if any(output.iterdir()): + raise ValueError(f"existing {kind} prelaunch output is not empty") + return expected_evaluator_packet(kind, identity, attempt) + runtime_packet = expected_evaluator_packet(kind, identity, attempt) + root.mkdir(parents=True) + shutil.copytree(source_packet, runtime_packet) + if byte_tree_digest(runtime_packet) != expected_source_packet_digest(kind, identity): + raise ValueError(f"{kind} runtime packet differs from frozen packet") + make_read_only(runtime_packet) + output.mkdir() + os.utime(root, (0, 0), follow_symlinks=False) + root.chmod(0o555) + return runtime_packet + + +def verify_evaluator_runtime( + kind: str, + identity: str, + attempt: int, + source_packet: Path, + output: Path, + *, + allow_invalid_output: bool = False, +) -> None: + if output != expected_evaluator_output(kind, identity, attempt): + raise ValueError(f"non-neutral {kind} output path") + root = output.parent + if root.is_symlink() or not root.is_dir() or root.stat().st_mode & 0o222: + raise ValueError(f"invalid {kind} runtime root") + actual_entries = {entry.name for entry in root.iterdir()} + if ( + (not allow_invalid_output and actual_entries != {"packet", "output"}) + or ( + allow_invalid_output + and ("packet" not in actual_entries or actual_entries - {"packet", "output"}) + ) + ): + raise ValueError(f"unexpected {kind} runtime inventory") + if not allow_invalid_output and ( + not output.exists() + or output.is_symlink() + or not stat.S_ISDIR(output.lstat().st_mode) + ): + raise ValueError(f"invalid {kind} output directory") + runtime_packet = expected_evaluator_packet(kind, identity, attempt) + if byte_tree_digest(source_packet) != expected_source_packet_digest(kind, identity): + raise ValueError(f"{kind} source packet changed") + if byte_tree_digest(runtime_packet) != expected_source_packet_digest(kind, identity): + raise ValueError(f"{kind} runtime packet changed") + + +def evaluator_runtime_forensics( + kind: str, + identity: str, + attempt: int, + source_packet: Path, + output: Path, + error: Exception, +) -> dict[str, Any]: + expected_output = expected_evaluator_output(kind, identity, attempt) + if output != expected_output: + raise ValueError(f"forensic evaluator output is not the neutral path {expected_output}") + root = output.parent + root_is_real = root.exists() and not root.is_symlink() and root.is_dir() + return { + "schema_version": 1, + "kind": kind, + "identity": identity, + "attempt": attempt, + "verification_error": f"{type(error).__name__}: {error}", + "expected": { + "output": str(expected_output), + "source_packet_byte_tree_sha256": expected_source_packet_digest(kind, identity), + }, + "observed": { + "source_packet": observe_path(source_packet), + "runtime_root": observe_path(root), + "packet": observe_path(root / "packet") + if root_is_real + else {"type": "unavailable"}, + "output": observe_path(output) + if root_is_real + else {"type": "unavailable"}, + }, + } + + +def validate_score(value: Any, mode: str, scorer: str) -> None: + if ( + not isinstance(value, dict) + or type(value.get("schema_version")) is not int + or value["schema_version"] != 1 + ): + raise ValueError("invalid score envelope") + if set(value) != {"schema_version", "mode", "scorer_id", "reports", "ambiguities"}: + raise ValueError("unexpected score fields") + if value.get("mode") != mode or value.get("scorer_id") != scorer: + raise ValueError("score mode or scorer mismatch") + if not isinstance(value["ambiguities"], list) or any( + not is_nonblank_string(item) for item in value["ambiguities"] + ): + raise ValueError("invalid score ambiguities") + reports = value.get("reports") + if ( + not isinstance(reports, list) + or len(reports) != 10 + or any(not isinstance(report, dict) for report in reports) + ): + raise ValueError("score must contain ten reports") + report_labels = [report.get("label") for report in reports] + if any(not isinstance(label, str) for label in report_labels) or tuple( + sorted(report_labels) + ) != LABELS: + raise ValueError("score labels must be exactly A-J") + expected_atoms = set(atom_ids(mode)) + allowed_hard = set(hard_error_ids(mode)) + for report in reports: + if set(report) != { + "label", + "atoms", + "hard_errors", + "proposal_laundering", + "tcb_authority_defect", + "visible_scope_defect", + "novel_findings", + }: + raise ValueError("unexpected report-score fields") + atoms = report.get("atoms") + if ( + not isinstance(atoms, list) + or any(not isinstance(atom, dict) for atom in atoms) + or {atom.get("id") for atom in atoms} != expected_atoms + ): + raise ValueError(f"{mode}/{report.get('label')}: incomplete atom set") + if len(atoms) != len(expected_atoms): + raise ValueError(f"{mode}/{report.get('label')}: duplicate atom ID") + for atom in atoms: + if set(atom) != {"id", "decision", "evidence"}: + raise ValueError("unexpected atom fields") + if ( + atom.get("decision") not in {"PASS", "FAIL"} + or not is_nonblank_string(atom.get("evidence")) + ): + raise ValueError("invalid atom decision") + hard = report.get("hard_errors") + if not isinstance(hard, list) or any( + not isinstance(finding, dict) for finding in hard + ): + raise ValueError("hard_errors must be a list") + hard_ids = [finding.get("id") for finding in hard] + if len(hard_ids) != len(set(hard_ids)) or not set(hard_ids) <= allowed_hard: + raise ValueError(f"invalid hard-error IDs: {hard_ids}") + if any( + not is_nonblank_string(finding.get("evidence")) + for finding in hard + ): + raise ValueError("hard error lacks evidence") + if any(set(finding) != {"id", "evidence"} for finding in hard): + raise ValueError("unexpected hard-error fields") + for field in ("proposal_laundering", "tcb_authority_defect", "visible_scope_defect"): + flag = report.get(field) + if not isinstance(flag, dict) or not isinstance(flag.get("present"), bool): + raise ValueError(f"invalid {field} flag") + if set(flag) != {"present", "evidence"} or not is_nonblank_string( + flag.get("evidence") + ): + raise ValueError(f"invalid {field} evidence") + novel = report.get("novel_findings") + if not isinstance(novel, list) or any( + not isinstance(finding, dict) for finding in novel + ): + raise ValueError("novel_findings must be a list") + novel_ids = [finding.get("id") for finding in novel] + if len(novel_ids) != len(set(novel_ids)) or any( + not re.fullmatch(r"N[1-9][0-9]*", str(identifier)) for identifier in novel_ids + ): + raise ValueError("invalid novel-finding IDs") + if any( + not is_nonblank_string(finding.get("evidence")) + for finding in novel + ): + raise ValueError("novel finding lacks evidence") + if any(set(finding) != {"id", "evidence"} for finding in novel): + raise ValueError("unexpected novel-finding fields") + + +def record_score( + mode: str, scorer: str, attempt: int, output: Path, agent_id: str +) -> None: + assert_evaluator_started("scorer", f"{mode}-{scorer}", attempt, agent_id) + verify_score_packet(mode, scorer) + verify_evaluator_runtime( + "scorer", + f"{mode}-{scorer}", + attempt, + SCORING / "packets" / mode / scorer, + output, + ) + source = output / "score.json" + entries = output_entries(output) + if [entry.name for entry in entries] != ["score.json"]: + raise ValueError("scorer output is not exactly score.json") + raw = source.read_bytes() + value = json.loads(raw.decode("utf-8")) + validate_score(value, mode, scorer) + attempt_directory = ( + SCORING + / "evaluator-attempts" + / "scoring" + / f"{mode}-{scorer}" + / str(attempt) + ) + entries = preserve_captured_file(attempt_directory / "raw-output", "score.json", raw) + write_once(attempt_directory / "raw-output-manifest.json", json_dump(entries)) + attempt_attestation = { + "schema_version": 1, + "phase": "scoring", + "kind": "scorer", + "identity": f"{mode}-{scorer}", + "attempt": attempt, + "agent_id": agent_id, + "disposition": "COMPLETE", + "recorded_utc": utc_now(), + } + write_once(attempt_directory / "attestation.json", json_dump(attempt_attestation)) + make_read_only(attempt_directory) + append_event( + "scoring", + "evaluator_attempt_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(attempt_directory), + details={"kind": "scorer", "identity": f"{mode}-{scorer}", "disposition": "COMPLETE"}, + ) + destination = SCORING / "raw" / mode / f"{scorer}.json" + write_bytes_once(destination, raw) + destination.chmod(0o444) + append_event( + "scoring", + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": "scorer", "identity": f"{mode}-{scorer}", "api_state": "COMPLETED"}, + ) + append_event( + "scoring", + "score_preserved", + attempt=attempt, + agent_id=agent_id, + digest=sha256_file(destination), + details={"mode": mode, "scorer": scorer}, + ) + print(destination) + + +def report_by_label(score: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {report["label"]: report for report in score["reports"]} + + +def preserved_digest(event_name: str, **details: str) -> str: + matches = [ + event + for event in event_records() + if event["event"] == event_name + and all(event.get("details", {}).get(key) == value for key, value in details.items()) + ] + if len(matches) != 1 or "sha256" not in matches[0]: + raise ValueError(f"expected one preserved digest for {event_name} {details}") + return matches[0]["sha256"] + + +def load_preserved_score(mode: str, scorer: str) -> dict[str, Any]: + path = SCORING / "raw" / mode / f"{scorer}.json" + expected = preserved_digest("score_preserved", mode=mode, scorer=scorer) + if sha256_file(path) != expected: + raise ValueError(f"raw score changed after preservation: {mode}/{scorer}") + value = json.loads(path.read_text()) + validate_score(value, mode, scorer) + return value + + +def evidence_for_hard(report: dict[str, Any], identifier: str) -> str: + for finding in report["hard_errors"]: + if finding["id"] == identifier: + return finding["evidence"] + return "No applicable hard error recorded." + + +def disagreement_cells(mode: str, s1: dict[str, Any], s2: dict[str, Any]) -> list[dict[str, Any]]: + cells: list[dict[str, Any]] = [] + by_scorer = {"s1": report_by_label(s1), "s2": report_by_label(s2)} + for label in LABELS: + first = by_scorer["s1"][label] + second = by_scorer["s2"][label] + atoms1 = {atom["id"]: atom for atom in first["atoms"]} + atoms2 = {atom["id"]: atom for atom in second["atoms"]} + for atom in atom_ids(mode): + if atoms1[atom]["decision"] != atoms2[atom]["decision"]: + cells.append( + { + "label": label, + "field": f"atom:{atom}", + "s1": {"decision": atoms1[atom]["decision"], "evidence": atoms1[atom]["evidence"]}, + "s2": {"decision": atoms2[atom]["decision"], "evidence": atoms2[atom]["evidence"]}, + } + ) + for identifier in hard_error_ids(mode): + present1 = any(item["id"] == identifier for item in first["hard_errors"]) + present2 = any(item["id"] == identifier for item in second["hard_errors"]) + if present1 != present2: + cells.append( + { + "label": label, + "field": f"hard_error:{identifier}", + "s1": {"decision": "PRESENT" if present1 else "ABSENT", "evidence": evidence_for_hard(first, identifier)}, + "s2": {"decision": "PRESENT" if present2 else "ABSENT", "evidence": evidence_for_hard(second, identifier)}, + } + ) + for field in ("proposal_laundering", "tcb_authority_defect", "visible_scope_defect"): + flag1 = first[field] + flag2 = second[field] + if flag1["present"] != flag2["present"]: + cells.append( + { + "label": label, + "field": field, + "s1": {"decision": "PRESENT" if flag1["present"] else "ABSENT", "evidence": flag1["evidence"]}, + "s2": {"decision": "PRESENT" if flag2["present"] else "ABSENT", "evidence": flag2["evidence"]}, + } + ) + for scorer, report, other in (("s1", first, "s2"), ("s2", second, "s1")): + for finding in report["novel_findings"]: + cells.append( + { + "label": label, + "field": f"novel:{scorer}:{finding['id']}", + scorer: {"decision": "PRESENT", "evidence": finding["evidence"]}, + other: {"decision": "ABSENT", "evidence": "Not independently proposed; adjudicate the candidate on its merits."}, + } + ) + keys = [(cell["label"], cell["field"]) for cell in cells] + if len(keys) != len(set(keys)): + raise ValueError("duplicate disagreement field") + return cells + + +def build_disagreements(mode: str) -> None: + scores: dict[str, dict[str, Any]] = {} + for scorer in SCORERS: + scores[scorer] = load_preserved_score(mode, scorer) + value = {"schema_version": 1, "mode": mode, "cells": disagreement_cells(mode, scores["s1"], scores["s2"])} + destination = SCORING / "disagreements" / f"{mode}.json" + write_once(destination, json_dump(value)) + destination.chmod(0o444) + append_event( + "adjudication", + "disagreements_materialized", + digest=sha256_file(destination), + details={"mode": mode, "count": len(value["cells"])}, + ) + print(f"{mode}: {len(value['cells'])} disputed or novel cells") + + +def build_adjudication_packet(mode: str) -> None: + disagreement_path = SCORING / "disagreements" / f"{mode}.json" + if sha256_file(disagreement_path) != preserved_digest("disagreements_materialized", mode=mode): + raise ValueError(f"disagreements changed before packet build: {mode}") + disagreements = json.loads(disagreement_path.read_text()) + if not disagreements["cells"]: + print(f"{mode}: no adjudication packet required") + return + packet = SCORING / "adjudication-packets" / mode + if packet.exists(): + raise FileExistsError(f"adjudication packet exists: {packet}") + (packet / "reports").mkdir(parents=True) + source_packet = SCORING / "packets" / mode / "s1" + verify_score_packet(mode, "s1") + source_packet_digest = preserved_digest("blind_packet_preserved", mode=mode, scorer="s1") + source_manifest = json.loads((source_packet / "PACKET.json").read_text()) + shutil.copytree(source_packet / "target", packet / "target") + for name in ("allowlist.txt", "SCORER.md", "RUBRIC.md"): + shutil.copy2(source_packet / name, packet / name) + shutil.copy2(FREEZE / "schemas" / "adjudication.schema.json", packet / "adjudication.schema.json") + shutil.copy2(disagreement_path, packet / "DISAGREEMENTS.json") + disputed_labels = sorted({cell["label"] for cell in disagreements["cells"]}) + for label in disputed_labels: + shutil.copy2(source_packet / "reports" / f"{label}.md", packet / "reports" / f"{label}.md") + if ( + sha256_file(packet / "reports" / f"{label}.md") + != source_manifest["report_sha256"][label] + ): + raise ValueError(f"adjudication report changed during copy: {mode}/{label}") + target_row = next(row for row in load_target_map().values() if row["mode"] == mode) + if tar_tree_digest(packet / "target") != target_row["tree_sha256"]: + raise ValueError(f"adjudication target lost frozen identity: {mode}") + if sha256_file(packet / "DISAGREEMENTS.json") != preserved_digest( + "disagreements_materialized", mode=mode + ): + raise ValueError(f"adjudication disagreements changed during copy: {mode}") + expected_copies = { + "allowlist.txt": "allowlist_sha256", + "SCORER.md": "common_rules_sha256", + "RUBRIC.md": "rubric_sha256", + } + for name, field in expected_copies.items(): + if sha256_file(packet / name) != source_manifest[field]: + raise ValueError(f"adjudication packet input changed during copy: {mode}/{name}") + if sha256_file(packet / "adjudication.schema.json") != frozen_file_digest( + FREEZE / "schemas" / "adjudication.schema.json" + ): + raise ValueError(f"adjudication schema changed during copy: {mode}") + manifest = { + "schema_version": 1, + "mode": mode, + "source_score_packet_sha256": source_packet_digest, + "disputed_labels": disputed_labels, + "disagreements_sha256": sha256_file(packet / "DISAGREEMENTS.json"), + "report_sha256": {label: sha256_file(packet / "reports" / f"{label}.md") for label in disputed_labels}, + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "adjudication.schema.json"), + } + write_once(packet / "PACKET.json", json_dump(manifest)) + make_read_only(packet) + append_event( + "adjudication", + "adjudication_packet_preserved", + digest=byte_tree_digest(packet), + details={"mode": mode, "source_score_packet_sha256": source_packet_digest}, + ) + print(packet) + + +def verify_adjudication_packet(mode: str) -> None: + packet = SCORING / "adjudication-packets" / mode + manifest = json.loads((packet / "PACKET.json").read_text()) + disagreements = json.loads((packet / "DISAGREEMENTS.json").read_text()) + if ( + type(manifest.get("schema_version")) is not int + or manifest["schema_version"] != 1 + or manifest.get("mode") != mode + or disagreements.get("mode") != mode + ): + raise ValueError(f"adjudication packet identity mismatch: {mode}") + verify_score_packet(mode, "s1") + source_digest = preserved_digest("blind_packet_preserved", mode=mode, scorer="s1") + if manifest.get("source_score_packet_sha256") != source_digest: + raise ValueError(f"adjudication packet source binding changed: {mode}") + checks = { + "disagreements_sha256": sha256_file(packet / "DISAGREEMENTS.json"), + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "adjudication.schema.json"), + } + for field, actual in checks.items(): + if manifest.get(field) != actual: + raise ValueError(f"adjudication packet changed: {mode}/{field}") + labels = manifest.get("disputed_labels") + expected_labels = sorted({cell["label"] for cell in disagreements.get("cells", [])}) + if not isinstance(labels, list) or labels != expected_labels: + raise ValueError(f"adjudication packet disputed-label inventory changed: {mode}") + report_hashes = { + label: sha256_file(packet / "reports" / f"{label}.md") for label in labels + } + if manifest.get("report_sha256") != report_hashes: + raise ValueError(f"adjudication reports changed: {mode}") + expected_files = { + "PACKET.json", + "DISAGREEMENTS.json", + "allowlist.txt", + "RUBRIC.md", + "SCORER.md", + "adjudication.schema.json", + *(f"reports/{label}.md" for label in labels), + } + expected_files.update( + path.relative_to(packet).as_posix() + for path in (packet / "target").rglob("*") + if path.is_file() + ) + actual_files = { + path.relative_to(packet).as_posix() for path in packet.rglob("*") if path.is_file() + } + if actual_files != expected_files: + raise ValueError(f"adjudication packet has unexpected files: {mode}") + expected_packet_digest = preserved_digest("adjudication_packet_preserved", mode=mode) + if byte_tree_digest(packet) != expected_packet_digest: + raise ValueError(f"adjudication packet differs from its external event: {mode}") + + +def validate_adjudication(value: Any, mode: str) -> None: + disagreement_path = SCORING / "disagreements" / f"{mode}.json" + expected_digest = preserved_digest("disagreements_materialized", mode=mode) + if sha256_file(disagreement_path) != expected_digest: + raise ValueError(f"disagreements changed after preservation: {mode}") + disagreements = json.loads(disagreement_path.read_text()) + expected = {(cell["label"], cell["field"]) for cell in disagreements["cells"]} + if ( + not isinstance(value, dict) + or type(value.get("schema_version")) is not int + or value["schema_version"] != 1 + or value.get("mode") != mode + ): + raise ValueError("invalid adjudication envelope") + if set(value) != {"schema_version", "mode", "decisions", "ambiguities"}: + raise ValueError("unexpected adjudication fields") + if not isinstance(value["ambiguities"], list) or any( + not is_nonblank_string(item) for item in value["ambiguities"] + ): + raise ValueError("invalid adjudication ambiguities") + decisions = value.get("decisions") + if not isinstance(decisions, list) or any( + not isinstance(decision, dict) for decision in decisions + ): + raise ValueError("adjudication decisions must be a list") + if any( + not isinstance(decision.get("label"), str) + or not isinstance(decision.get("field"), str) + for decision in decisions + ): + raise ValueError("adjudication decision identity is invalid") + actual = {(decision.get("label"), decision.get("field")) for decision in decisions} + if actual != expected or len(decisions) != len(expected): + raise ValueError("adjudication does not resolve exactly every disputed cell") + for decision in decisions: + if set(decision) != {"label", "field", "decision", "evidence"}: + raise ValueError("unexpected adjudication-decision fields") + expected_values = {"PASS", "FAIL"} if decision["field"].startswith("atom:") else {"PRESENT", "ABSENT"} + if ( + decision.get("decision") not in expected_values + or not is_nonblank_string(decision.get("evidence")) + ): + raise ValueError("invalid adjudication decision") + + +def synthetic_score(mode: str, scorer: str) -> dict[str, Any]: + reports: list[dict[str, Any]] = [] + for label in LABELS: + reports.append( + { + "label": label, + "atoms": [ + {"id": atom, "decision": "PASS", "evidence": "Synthetic complete evidence."} + for atom in atom_ids(mode) + ], + "hard_errors": [], + "proposal_laundering": {"present": False, "evidence": "No proposal laundering."}, + "tcb_authority_defect": {"present": False, "evidence": "No TCB or authority defect."}, + "visible_scope_defect": {"present": False, "evidence": "No visible source-scope defect."}, + "novel_findings": [], + } + ) + return { + "schema_version": 1, + "mode": mode, + "scorer_id": scorer, + "reports": reports, + "ambiguities": [], + } + + +def self_test() -> None: + validate_static(require_lock=False) + for mode in MODES: + first = synthetic_score(mode, "s1") + second = synthetic_score(mode, "s2") + validate_score(first, mode, "s1") + validate_score(second, mode, "s2") + first_report = first["reports"][0] + second_report = second["reports"][0] + first_report["atoms"][0]["decision"] = "FAIL" + first_report["hard_errors"] = [ + {"id": hard_error_ids(mode)[0], "evidence": "Synthetic hard-error evidence."} + ] + first_report["proposal_laundering"] = {"present": True, "evidence": "Synthetic flag evidence."} + first_report["novel_findings"] = [{"id": "N1", "evidence": "Synthetic novel candidate."}] + cells = disagreement_cells(mode, first, second) + expected_fields = { + f"atom:{atom_ids(mode)[0]}", + f"hard_error:{hard_error_ids(mode)[0]}", + "proposal_laundering", + "novel:s1:N1", + } + if {cell["field"] for cell in cells} != expected_fields: + raise AssertionError(f"{mode}: disagreement self-test failed") + invalid = synthetic_score(mode, "s1") + invalid["reports"][1]["label"] = "A" + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: duplicate label was accepted") + invalid = synthetic_score(mode, "s1") + invalid["reports"][0]["atoms"].pop() + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: incomplete atom set was accepted") + invalid = synthetic_score(mode, "s1") + invalid["schema_version"] = True + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: boolean schema version was accepted") + invalid = synthetic_score(mode, "s1") + invalid["reports"][0]["atoms"][0]["evidence"] = 1 + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: non-string evidence was accepted") + invalid = synthetic_score(mode, "s1") + invalid["reports"][0]["atoms"][0]["evidence"] = " " + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: whitespace-only evidence was accepted") + schedule = load_schedule() + for run_id, row in schedule.items(): + runtime = Path("/tmp/ur-eval") / row["cell_id"] + prompt = render_report_prompt(run_id, runtime) + if "[" + "PACKAGE]" in prompt or "[" + "WORD_LIMIT]" in prompt: + raise AssertionError("unresolved report prompt") + if report_reminder_text() != ( + "Complete now within the frozen word limit using only material already\n" + "inspected; do not widen scope." + ): + raise AssertionError("frozen reminder extraction changed") + if expected_evaluator_output("scorer", "S-s1", 1) == expected_evaluator_output( + "scorer", "S-s1", 2 + ): + raise AssertionError("evaluator attempts share a runtime path") + assert_evaluator_attempt_allowed("scorer", "S-s1", 1, []) + try: + assert_evaluator_attempt_allowed("scorer", "S-s1", 2, []) + except ValueError: + pass + else: + raise AssertionError("evaluator retry without infrastructure failure was accepted") + assert_evaluator_attempt_allowed( + "scorer", + "S-s1", + 2, + [ + { + "event": "evaluator_infrastructure_failure", + "attempt": 1, + "details": {"kind": "scorer", "identity": "S-s1"}, + } + ], + ) + invalid_event = { + "schema_version": True, + "sequence": 1, + "previous_event_sha256": None, + "time_utc": utc_now(), + "phase": "freeze", + "event": "synthetic", + "details": {}, + } + try: + validate_event_record(invalid_event, 1) + except ValueError: + pass + else: + raise AssertionError("boolean event schema version was accepted") + metadata_root = Path(tempfile.mkdtemp(prefix="ur-packet-self-test-", dir="/tmp")) + try: + nested = metadata_root / "nested" + nested.mkdir() + file = nested / "report.md" + file.write_text("packet metadata self-test\n") + make_read_only(metadata_root) + for item in (metadata_root, nested, file): + if item.stat().st_mtime_ns != 0: + raise AssertionError(f"metadata timestamp was not normalized: {item}") + finally: + for item in sorted(metadata_root.rglob("*"), reverse=True): + item.chmod(0o755 if item.is_dir() else 0o644) + metadata_root.chmod(0o755) + shutil.rmtree(metadata_root) + print("protocol self-test passed") + + +def record_adjudication( + mode: str, attempt: int, output: Path, agent_id: str +) -> None: + assert_evaluator_started("adjudicator", mode, attempt, agent_id) + verify_adjudication_packet(mode) + verify_evaluator_runtime( + "adjudicator", + mode, + attempt, + SCORING / "adjudication-packets" / mode, + output, + ) + entries = output_entries(output) + if [entry.name for entry in entries] != ["adjudication.json"]: + raise ValueError("adjudicator output is not exactly adjudication.json") + raw = entries[0].read_bytes() + value = json.loads(raw.decode("utf-8")) + validate_adjudication(value, mode) + attempt_directory = ( + SCORING + / "evaluator-attempts" + / "adjudication" + / mode + / str(attempt) + ) + raw_entries = preserve_captured_file( + attempt_directory / "raw-output", "adjudication.json", raw + ) + write_once( + attempt_directory / "raw-output-manifest.json", json_dump(raw_entries) + ) + attempt_attestation = { + "schema_version": 1, + "phase": "adjudication", + "kind": "adjudicator", + "identity": mode, + "attempt": attempt, + "agent_id": agent_id, + "disposition": "COMPLETE", + "recorded_utc": utc_now(), + } + write_once(attempt_directory / "attestation.json", json_dump(attempt_attestation)) + make_read_only(attempt_directory) + append_event( + "adjudication", + "evaluator_attempt_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(attempt_directory), + details={"kind": "adjudicator", "identity": mode, "disposition": "COMPLETE"}, + ) + destination = SCORING / "adjudications" / f"{mode}.json" + write_bytes_once(destination, raw) + destination.chmod(0o444) + append_event( + "adjudication", + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": "adjudicator", "identity": mode, "api_state": "COMPLETED"}, + ) + append_event( + "adjudication", + "adjudication_preserved", + attempt=attempt, + agent_id=agent_id, + digest=sha256_file(destination), + details={"mode": mode}, + ) + print(destination) + + +def record_invalid_evaluator( + phase: str, + identity: str, + attempt: int, + output: Path, + agent_id: str, + evidence: str, +) -> None: + invalid_marker = RUN / "INVALID.json" + if invalid_marker.exists(): + raise FileExistsError("run is already marked INVALID") + if not evidence.strip(): + raise ValueError("invalid evaluator output requires nonempty evidence") + kind = "scorer" if phase == "scoring" else "adjudicator" + assert_evaluator_started(kind, identity, attempt, agent_id) + if kind == "scorer": + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + else: + source_packet = SCORING / "adjudication-packets" / identity + forensics: dict[str, Any] | None = None + try: + verify_evaluator_runtime( + kind, identity, attempt, source_packet, output, allow_invalid_output=True + ) + except (OSError, ValueError) as error: + forensics = evaluator_runtime_forensics( + kind, identity, attempt, source_packet, output, error + ) + evidence = ( + f"{evidence} Runtime/packet verification failed: " + f"{forensics['verification_error']}" + ) + destination = SCORING / "invalid" / phase / identity / str(attempt) + root = output.parent + if forensics is None: + entries = snapshot_path(output, destination / "raw-output") + write_once(destination / "raw-output-manifest.json", json_dump(entries)) + else: + entries = snapshot_path(root, destination / "raw-runtime") + write_once(destination / "raw-runtime-manifest.json", json_dump(entries)) + source_entries = snapshot_path( + source_packet, destination / "source-packet-at-verification" + ) + write_once( + destination / "source-packet-at-verification-manifest.json", + json_dump(source_entries), + ) + if forensics is not None: + write_once(destination / "runtime-forensics.json", json_dump(forensics)) + attestation = { + "schema_version": 1, + "phase": phase, + "identity": identity, + "attempt": attempt, + "agent_id": agent_id, + "evidence": evidence, + "disposition": "INVALID_NONRERUNNABLE_EVALUATOR_OUTPUT", + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(attestation)) + make_read_only(destination) + append_event( + phase, + "invalid_output_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"kind": kind, "identity": identity}, + ) + write_once(invalid_marker, json_dump(attestation)) + invalid_marker.chmod(0o444) + append_event( + phase, + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": kind, "identity": identity, "api_state": "INVALID_OUTPUT"}, + ) + append_event( + phase, + "run_invalidated", + attempt=attempt, + agent_id=agent_id, + digest=sha256_file(invalid_marker), + details={"identity": identity, "evidence": evidence}, + ) + print(invalid_marker) + + +def preserve_failed_evaluator_attempt( + phase: str, + identity: str, + attempt: int, + output: Path, + agent_id: str, + disposition: str, + evidence: str, +) -> None: + if disposition not in INFRA_FAILURE_CODES: + raise ValueError(f"non-infrastructure evaluator disposition: {disposition}") + if not evidence.strip(): + raise ValueError("evaluator infrastructure failure requires nonempty evidence") + kind = "scorer" if phase == "scoring" else "adjudicator" + assert_evaluator_started(kind, identity, attempt, agent_id) + if kind == "scorer": + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + else: + source_packet = SCORING / "adjudication-packets" / identity + verify_evaluator_runtime( + kind, identity, attempt, source_packet, output, allow_invalid_output=True + ) + destination = SCORING / "evaluator-attempts" / phase / identity / str(attempt) + entries = snapshot_path(output, destination / "raw-output") + write_once(destination / "raw-output-manifest.json", json_dump(entries)) + attestation = { + "schema_version": 1, + "phase": phase, + "kind": kind, + "identity": identity, + "attempt": attempt, + "agent_id": agent_id, + "evidence": evidence, + "disposition": disposition, + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(attestation)) + make_read_only(destination) + append_event( + phase, + "evaluator_attempt_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"kind": kind, "identity": identity, "disposition": disposition}, + ) + append_event( + phase, + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": kind, "identity": identity, "api_state": "INFRASTRUCTURE_FAILURE"}, + ) + append_event( + phase, + "evaluator_infrastructure_failure", + attempt=attempt, + agent_id=agent_id, + details={ + "kind": kind, + "identity": identity, + "disposition": disposition, + "evidence": evidence, + }, + ) + print(json_dump(attestation), end="") + + +def decision_lookup(value: dict[str, Any]) -> dict[tuple[str, str], dict[str, Any]]: + return {(item["label"], item["field"]): item for item in value.get("decisions", [])} + + +def merge_final(mode: str) -> None: + scores = {scorer: load_preserved_score(mode, scorer) for scorer in SCORERS} + disagreement_path = SCORING / "disagreements" / f"{mode}.json" + if sha256_file(disagreement_path) != preserved_digest( + "disagreements_materialized", mode=mode + ): + raise ValueError(f"disagreements changed after preservation: {mode}") + disagreements = json.loads(disagreement_path.read_text()) + if disagreements["cells"]: + adjudication = json.loads((SCORING / "adjudications" / f"{mode}.json").read_text()) + expected_adjudication = preserved_digest("adjudication_preserved", mode=mode) + if sha256_file(SCORING / "adjudications" / f"{mode}.json") != expected_adjudication: + raise ValueError(f"adjudication changed after preservation: {mode}") + validate_adjudication(adjudication, mode) + decisions = decision_lookup(adjudication) + else: + decisions = {} + by_scorer = {scorer: report_by_label(score) for scorer, score in scores.items()} + final_reports: list[dict[str, Any]] = [] + confirmed_novel: list[dict[str, Any]] = [] + for label in LABELS: + first = by_scorer["s1"][label] + second = by_scorer["s2"][label] + atoms: dict[str, str] = {} + atoms1 = {atom["id"]: atom for atom in first["atoms"]} + atoms2 = {atom["id"]: atom for atom in second["atoms"]} + for atom in atom_ids(mode): + if atoms1[atom]["decision"] == atoms2[atom]["decision"]: + atoms[atom] = atoms1[atom]["decision"] + else: + atoms[atom] = decisions[(label, f"atom:{atom}")]["decision"] + final_hard: list[str] = [] + for identifier in hard_error_ids(mode): + present = [any(item["id"] == identifier for item in report["hard_errors"]) for report in (first, second)] + if present[0] == present[1]: + chosen = present[0] + else: + chosen = decisions[(label, f"hard_error:{identifier}")]["decision"] == "PRESENT" + if chosen: + final_hard.append(identifier) + flags: dict[str, bool] = {} + for field in ("proposal_laundering", "tcb_authority_defect", "visible_scope_defect"): + present = [first[field]["present"], second[field]["present"]] + if present[0] == present[1]: + flags[field] = present[0] + else: + flags[field] = decisions[(label, field)]["decision"] == "PRESENT" + final_reports.append({"label": label, "atoms": atoms, "hard_errors": final_hard, **flags}) + for scorer, report in (("s1", first), ("s2", second)): + for finding in report["novel_findings"]: + field = f"novel:{scorer}:{finding['id']}" + if decisions[(label, field)]["decision"] == "PRESENT": + confirmed_novel.append({"label": label, "field": field, "evidence": decisions[(label, field)]["evidence"]}) + value = {"schema_version": 1, "mode": mode, "reports": final_reports, "confirmed_novel_findings": confirmed_novel} + destination = SCORING / "final" / f"{mode}.json" + write_once(destination, json_dump(value)) + destination.chmod(0o444) + append_event("adjudication", "final_blind_score_locked", digest=sha256_file(destination), details={"mode": mode}) + print(destination) + + +def aggregate() -> None: + schedule = load_schedule() + conditions = load_condition_map() + targets = load_target_map() + blind = load_blind_map() + index = load_index(require_collection_lock=True) + counts: dict[str, dict[str, dict[str, int]]] = {} + defects: dict[str, dict[str, list[dict[str, Any]]]] = {} + failures: list[dict[str, Any]] = [] + novel: list[dict[str, Any]] = [] + final_digests: dict[str, str] = {} + for mode in MODES: + path = SCORING / "final" / f"{mode}.json" + expected_final = preserved_digest("final_blind_score_locked", mode=mode) + if sha256_file(path) != expected_final: + raise ValueError(f"final blind score changed after lock: {mode}") + value = json.loads(path.read_text()) + final_digests[mode] = sha256_file(path) + reports = {report["label"]: report for report in value["reports"]} + if tuple(sorted(reports)) != LABELS: + raise ValueError(f"{mode} final labels incomplete") + counts[mode] = {role: {atom: 0 for atom in atom_ids(mode)} for role in prepare.CONDITIONS} + defects[mode] = {role: [] for role in prepare.CONDITIONS} + for label, run_id in blind[mode].items(): + row = schedule[run_id] + if targets[row["target_label"]]["mode"] != mode: + raise ValueError("unblinding mode mismatch") + role = conditions[row["condition_label"]]["role"] + report = reports[label] + for atom, decision in report["atoms"].items(): + if decision == "PASS": + counts[mode][role][atom] += 1 + else: + failures.append({"mode": mode, "role": role, "run_id": run_id, "label": label, "atom": atom}) + over_budget = index[run_id]["word_count"] > index[run_id]["word_cap"] + flags = { + "hard_errors": report["hard_errors"], + "proposal_laundering": report["proposal_laundering"], + "tcb_authority_defect": report["tcb_authority_defect"], + "visible_scope_defect": report["visible_scope_defect"], + "operational_scope_deviation": index[run_id]["operational_scope_deviation"], + "word_budget_defect": over_budget, + "semantic_noncompletion": index[run_id]["semantic_noncompletion"], + } + if flags["hard_errors"] or any( + flags[name] + for name in ( + "proposal_laundering", + "tcb_authority_defect", + "visible_scope_defect", + "operational_scope_deviation", + "word_budget_defect", + "semantic_noncompletion", + ) + ): + defects[mode][role].append({"run_id": run_id, "label": label, **flags}) + for finding in value["confirmed_novel_findings"]: + novel.append({"mode": mode, **finding}) + v3_failures = [failure for failure in failures if failure["role"] == "v3"] + v3_defects = [item for mode in MODES for item in defects[mode]["v3"]] + gate = { + "all_v3_atoms_5_of_5": not v3_failures, + "zero_v3_hard_errors": not any(item["hard_errors"] for item in v3_defects), + "zero_v3_proposal_laundering": not any(item["proposal_laundering"] for item in v3_defects), + "zero_v3_tcb_authority_defects": not any(item["tcb_authority_defect"] for item in v3_defects), + "zero_v3_semantic_noncompletion": not any( + item["semantic_noncompletion"] for item in v3_defects + ), + "zero_v3_scope_budget_defects": not any( + item["visible_scope_defect"] + or item["operational_scope_deviation"] + or item["word_budget_defect"] + for item in v3_defects + ), + } + gate["overall"] = all(gate.values()) + diagnostic_comparisons: list[dict[str, Any]] = [] + for mode in MODES: + for atom in atom_ids(mode): + v3_count = counts[mode]["v3"][atom] + v2_count = counts[mode]["v2"][atom] + if v3_count < v2_count: + classification = "V3_BELOW_V2" + elif v3_count == 5 and v2_count < 5: + classification = "TARGETED_LIFT_EVIDENCE" + elif v3_count == 5 and v2_count == 5: + classification = "CEILING_REPLICATION" + elif v3_count > v2_count: + classification = "V3_HIGHER_BUT_CONFIRMATION_FAILED" + else: + classification = "MATCHED_BELOW_CEILING" + diagnostic_comparisons.append( + { + "mode": mode, + "atom": atom, + "v3_passes": v3_count, + "v2_passes": v2_count, + "classification": classification, + } + ) + result = { + "schema_version": 1, + "unblinded_utc": utc_now(), + "condition_map": {label: row["role"] for label, row in conditions.items()}, + "final_score_sha256": final_digests, + "counts": counts, + "defects": defects, + "failed_atom_cells": failures, + "confirmed_novel_findings": novel, + "diagnostic_comparison": { + "any_v3_below_v2": any( + item["classification"] == "V3_BELOW_V2" + for item in diagnostic_comparisons + ), + "causal_claim": False, + "reason": "The coherent V3 and V2 packages differ in more than one isolated instruction.", + "atoms": diagnostic_comparisons, + }, + "v3_gate": gate, + } + RESULTS.mkdir(parents=True, exist_ok=True) + write_once(RUN / "unblinding.json", json_dump({"schema_version": 1, "unblinded_utc": result["unblinded_utc"], "condition_map": result["condition_map"], "final_score_sha256": final_digests})) + write_once(RESULTS / "aggregate.json", json_dump(result)) + lines = [ + "# V3 Targeted Confirmation Results", + "", + f"**Primary V3 gate: {'PASS' if gate['overall'] else 'FAIL'}.**", + "", + "Each atom cell is a pass count out of five. V3 is the confirmatory candidate; V2 is diagnostic.", + "", + "| Mode | Condition | Atom pass counts | Defective reports |", + "|---|---|---|---:|", + ] + for mode in MODES: + for role in ("v3", "v2"): + rendered = "; ".join(f"{atom} {count}/5" for atom, count in counts[mode][role].items()) + lines.append(f"| {mode} | {role.upper()} | {rendered} | {len(defects[mode][role])} |") + lines.extend( + [ + "", + "## Diagnostic comparison", + "", + f"Any V3 atom below matched V2: {'YES' if result['diagnostic_comparison']['any_v3_below_v2'] else 'NO'}.", + "", + "`TARGETED_LIFT_EVIDENCE` means V3 passed 5/5 while matched V2 was lower. " + "`CEILING_REPLICATION` means both passed 5/5. These coherent packages differ " + "in more than one isolated instruction, so no classification is causal proof.", + "", + "| Mode | Atom | V3 | V2 | Classification |", + "|---|---|---:|---:|---|", + ] + ) + for item in diagnostic_comparisons: + lines.append( + f"| {item['mode']} | {item['atom']} | {item['v3_passes']}/5 | " + f"{item['v2_passes']}/5 | {item['classification']} |" + ) + lines.extend(["", "## Primary gates", "", "| Gate | Result |", "|---|---|"]) + for name, passed in gate.items(): + if name != "overall": + lines.append(f"| {name.replace('_', ' ')} | {'PASS' if passed else 'FAIL'} |") + lines.extend(["", "## Integrity limitations", "", "Filesystem and URL isolation were procedural on a shared host. Exact hosted model-build and sampling-seed metadata were unavailable. Results are source-review capability observations under those constraints.", ""]) + write_once(RESULTS / "summary.md", "\n".join(lines)) + append_event("unblinding", "conditions_revealed", digest=sha256_file(RUN / "unblinding.json"), details={}) + append_event( + "result", + "aggregate_written", + digest=sha256_file(RESULTS / "aggregate.json"), + details={ + "v3_gate": gate["overall"], + "summary_sha256": sha256_file(RESULTS / "summary.md"), + }, + ) + print(RESULTS / "summary.md") + + +def record_freeze_lock() -> None: + if any(event["event"] == "freeze_locked" for event in event_records()): + raise ValueError("freeze lock was already recorded") + append_event( + "freeze", + "freeze_locked", + digest=sha256_file(FILE_MANIFEST), + details={"lock_sha256": sha256_file(LOCK)}, + ) + print("freeze lock recorded") + + +def assert_freeze_locked() -> None: + matches = [event for event in event_records() if event["event"] == "freeze_locked"] + if len(matches) != 1: + raise ValueError("evaluation operations require exactly one freeze-lock event") + event = matches[0] + if ( + event.get("sha256") != sha256_file(FILE_MANIFEST) + or event.get("details", {}).get("lock_sha256") != sha256_file(LOCK) + ): + raise ValueError("freeze-lock event does not bind the current lock") + + +def main() -> None: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + verify = sub.add_parser("verify-static") + verify.add_argument("--locked", action="store_true") + sub.add_parser("self-test") + sub.add_parser("write-file-manifest") + sub.add_parser("record-freeze-lock") + prepare_cell_parser = sub.add_parser("prepare-cell") + prepare_cell_parser.add_argument("run_id") + prepare_cell_parser.add_argument("runtime", type=Path) + report_prompt = sub.add_parser("report-prompt") + report_prompt.add_argument("run_id") + report_prompt.add_argument("runtime", type=Path) + record_report_parser = sub.add_parser("record-report") + record_report_parser.add_argument("run_id") + record_report_parser.add_argument("attempt", type=positive_int) + record_report_parser.add_argument("runtime", type=Path) + record_report_parser.add_argument("agent_id") + record_report_parser.add_argument("--scope-deviation", action="store_true") + record_report_parser.add_argument( + "--scope-evidence", default="No known operational source-scope deviation." + ) + failed_report_parser = sub.add_parser("record-failed-report") + failed_report_parser.add_argument("run_id") + failed_report_parser.add_argument("attempt", type=positive_int) + failed_report_parser.add_argument("runtime", type=Path) + failed_report_parser.add_argument("agent_id") + failed_report_parser.add_argument( + "disposition", choices=sorted(INFRA_FAILURE_CODES | TERMINAL_REPORT_FAILURE_CODES) + ) + failed_report_parser.add_argument("evidence") + failed_report_parser.add_argument("--scope-deviation", action="store_true") + agent_start_parser = sub.add_parser("agent-start") + agent_start_parser.add_argument("run_id") + agent_start_parser.add_argument("attempt", type=positive_int) + agent_start_parser.add_argument("agent_id") + prelaunch_parser = sub.add_parser("record-prelaunch-failure") + prelaunch_parser.add_argument("run_id") + prelaunch_parser.add_argument("evidence") + reminder_parser = sub.add_parser("reminder-text") + reminder_parser.add_argument("run_id") + reminder_parser.add_argument("attempt", type=positive_int) + reminder_parser.add_argument("agent_id") + sub.add_parser("build-score-packets") + scorer_prompt = sub.add_parser("scorer-prompt") + scorer_prompt.add_argument("mode", choices=MODES) + scorer_prompt.add_argument("scorer", choices=SCORERS) + scorer_prompt.add_argument("attempt", type=positive_int) + scorer_prompt.add_argument("output", type=Path) + record_score_parser = sub.add_parser("record-score") + record_score_parser.add_argument("mode", choices=MODES) + record_score_parser.add_argument("scorer", choices=SCORERS) + record_score_parser.add_argument("attempt", type=positive_int) + record_score_parser.add_argument("output", type=Path) + record_score_parser.add_argument("agent_id") + invalid_score_parser = sub.add_parser("record-invalid-score") + invalid_score_parser.add_argument("mode", choices=MODES) + invalid_score_parser.add_argument("scorer", choices=SCORERS) + invalid_score_parser.add_argument("attempt", type=positive_int) + invalid_score_parser.add_argument("output", type=Path) + invalid_score_parser.add_argument("agent_id") + invalid_score_parser.add_argument("evidence") + disagreements_parser = sub.add_parser("build-disagreements") + disagreements_parser.add_argument("mode", choices=MODES) + adjudication_packet = sub.add_parser("build-adjudication-packet") + adjudication_packet.add_argument("mode", choices=MODES) + adjudicator_prompt = sub.add_parser("adjudicator-prompt") + adjudicator_prompt.add_argument("mode", choices=MODES) + adjudicator_prompt.add_argument("attempt", type=positive_int) + adjudicator_prompt.add_argument("output", type=Path) + record_adjudication_parser = sub.add_parser("record-adjudication") + record_adjudication_parser.add_argument("mode", choices=MODES) + record_adjudication_parser.add_argument("attempt", type=positive_int) + record_adjudication_parser.add_argument("output", type=Path) + record_adjudication_parser.add_argument("agent_id") + invalid_adjudication_parser = sub.add_parser("record-invalid-adjudication") + invalid_adjudication_parser.add_argument("mode", choices=MODES) + invalid_adjudication_parser.add_argument("attempt", type=positive_int) + invalid_adjudication_parser.add_argument("output", type=Path) + invalid_adjudication_parser.add_argument("agent_id") + invalid_adjudication_parser.add_argument("evidence") + evaluator_start_parser = sub.add_parser("evaluator-start") + evaluator_start_parser.add_argument("kind", choices=("scorer", "adjudicator")) + evaluator_start_parser.add_argument("identity") + evaluator_start_parser.add_argument("attempt", type=positive_int) + evaluator_start_parser.add_argument("agent_id") + evaluator_prelaunch = sub.add_parser("record-evaluator-prelaunch-failure") + evaluator_prelaunch.add_argument("kind", choices=("scorer", "adjudicator")) + evaluator_prelaunch.add_argument("identity") + evaluator_prelaunch.add_argument("attempt", type=positive_int) + evaluator_prelaunch.add_argument("output", type=Path) + evaluator_prelaunch.add_argument("evidence") + failed_evaluator = sub.add_parser("record-failed-evaluator") + failed_evaluator.add_argument("kind", choices=("scorer", "adjudicator")) + failed_evaluator.add_argument("identity") + failed_evaluator.add_argument("attempt", type=positive_int) + failed_evaluator.add_argument("output", type=Path) + failed_evaluator.add_argument("agent_id") + failed_evaluator.add_argument("disposition", choices=sorted(INFRA_FAILURE_CODES)) + failed_evaluator.add_argument("evidence") + merge_parser = sub.add_parser("merge-final") + merge_parser.add_argument("mode", choices=MODES) + sub.add_parser("aggregate") + args = parser.parse_args() + + operational_commands = { + "record-freeze-lock", + "prepare-cell", + "report-prompt", + "agent-start", + "record-prelaunch-failure", + "reminder-text", + "record-report", + "record-failed-report", + "build-score-packets", + "scorer-prompt", + "record-score", + "record-invalid-score", + "build-disagreements", + "build-adjudication-packet", + "adjudicator-prompt", + "record-adjudication", + "record-invalid-adjudication", + "evaluator-start", + "record-evaluator-prelaunch-failure", + "record-failed-evaluator", + "merge-final", + "aggregate", + } + operation_lock_handle = None + if args.command in operational_commands: + operation_lock_handle = acquire_operation_lock() + terminal_invalid_event = any( + event["event"] in {"invalid_output_preserved", "run_invalidated"} + for event in event_records() + ) + invalid_marker = RUN / "INVALID.json" + if invalid_marker.exists() or invalid_marker.is_symlink() or terminal_invalid_event: + raise SystemExit("run is INVALID; no further evaluation command is permitted") + validate_static(require_lock=True, announce=False) + if args.command == "record-freeze-lock": + if any(event["event"] == "freeze_locked" for event in event_records()): + raise SystemExit("freeze lock was already recorded") + else: + assert_freeze_locked() + + if args.command == "verify-static": + validate_static(args.locked) + elif args.command == "self-test": + self_test() + elif args.command == "write-file-manifest": + if FILE_MANIFEST.exists(): + raise FileExistsError(FILE_MANIFEST) + write_once(FILE_MANIFEST, render_file_manifest()) + print(f"{FILE_MANIFEST.relative_to(RUN)} {sha256_file(FILE_MANIFEST)}") + elif args.command == "record-freeze-lock": + record_freeze_lock() + elif args.command == "prepare-cell": + prepare_cell(args.run_id, args.runtime) + elif args.command == "report-prompt": + verify_runtime(args.run_id, args.runtime) + print(render_report_prompt(args.run_id, args.runtime), end="") + elif args.command == "record-report": + record_report( + args.run_id, + args.attempt, + args.runtime, + args.agent_id, + args.scope_deviation, + args.scope_evidence, + ) + elif args.command == "record-failed-report": + preserve_failed_report_attempt( + args.run_id, + args.attempt, + args.runtime, + args.agent_id, + args.disposition, + args.evidence, + args.scope_deviation, + ) + elif args.command == "agent-start": + record_agent_start(args.run_id, args.attempt, args.agent_id) + elif args.command == "record-prelaunch-failure": + record_prelaunch_failure(args.run_id, args.evidence) + elif args.command == "reminder-text": + record_reminder(args.run_id, args.attempt, args.agent_id) + print(report_reminder_text(), end="") + elif args.command == "build-score-packets": + build_score_packets() + elif args.command == "scorer-prompt": + source_packet = SCORING / "packets" / args.mode / args.scorer + verify_score_packet(args.mode, args.scorer) + packet = prepare_evaluator_runtime( + "scorer", + f"{args.mode}-{args.scorer}", + args.attempt, + source_packet, + args.output, + ) + print( + render_packet_prompt("scorer.md", packet, args.output, SCORER_ID=args.scorer), + end="", + ) + elif args.command == "record-score": + record_score(args.mode, args.scorer, args.attempt, args.output, args.agent_id) + elif args.command == "record-invalid-score": + record_invalid_evaluator( + "scoring", + f"{args.mode}-{args.scorer}", + args.attempt, + args.output, + args.agent_id, + args.evidence, + ) + elif args.command == "build-disagreements": + build_disagreements(args.mode) + elif args.command == "build-adjudication-packet": + build_adjudication_packet(args.mode) + elif args.command == "adjudicator-prompt": + source_packet = SCORING / "adjudication-packets" / args.mode + verify_adjudication_packet(args.mode) + packet = prepare_evaluator_runtime( + "adjudicator", args.mode, args.attempt, source_packet, args.output + ) + print(render_packet_prompt("adjudicator.md", packet, args.output), end="") + elif args.command == "record-adjudication": + record_adjudication(args.mode, args.attempt, args.output, args.agent_id) + elif args.command == "record-invalid-adjudication": + record_invalid_evaluator( + "adjudication", + args.mode, + args.attempt, + args.output, + args.agent_id, + args.evidence, + ) + elif args.command == "evaluator-start": + record_evaluator_start(args.kind, args.identity, args.attempt, args.agent_id) + elif args.command == "record-evaluator-prelaunch-failure": + record_evaluator_prelaunch_failure( + args.kind, args.identity, args.attempt, args.output, args.evidence + ) + elif args.command == "record-failed-evaluator": + preserve_failed_evaluator_attempt( + "scoring" if args.kind == "scorer" else "adjudication", + args.identity, + args.attempt, + args.output, + args.agent_id, + args.disposition, + args.evidence, + ) + elif args.command == "merge-final": + merge_final(args.mode) + elif args.command == "aggregate": + aggregate() + + +if __name__ == "__main__": + main() diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/blind-map.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/blind-map.tsv new file mode 100644 index 0000000000..570f3aabda --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/blind-map.tsv @@ -0,0 +1,81 @@ +mode label run_id +S A r035 +S B r075 +S C r027 +S D r025 +S E r059 +S F r014 +S G r074 +S H r002 +S I r054 +S J r036 +C A r030 +C B r004 +C C r046 +C D r079 +C E r077 +C F r023 +C G r010 +C H r043 +C I r051 +C J r062 +X A r066 +X B r006 +X C r018 +X D r015 +X E r022 +X F r042 +X G r049 +X H r034 +X I r078 +X J r064 +Q A r008 +Q B r068 +Q C r060 +Q D r026 +Q E r048 +Q F r056 +Q G r031 +Q H r045 +Q I r001 +Q J r065 +W A r070 +W B r032 +W C r024 +W D r041 +W E r050 +W F r053 +W G r009 +W H r012 +W I r044 +W J r076 +M A r013 +M B r067 +M C r071 +M D r047 +M E r021 +M F r003 +M G r063 +M H r037 +M I r057 +M J r019 +R A r069 +R B r038 +R C r028 +R D r011 +R E r052 +R F r007 +R G r040 +R H r072 +R I r055 +R J r029 +K A r020 +K B r016 +K C r005 +K D r039 +K E r073 +K F r033 +K G r017 +K H r058 +K I r080 +K J r061 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/condition-map.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/condition-map.tsv new file mode 100644 index 0000000000..5afacde848 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/condition-map.tsv @@ -0,0 +1,3 @@ +condition_label role package_path tree_sha256 skill_sha256 +c0 v2 frozen-packages/40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897 40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897 a0a75ef8a14497aa78b50b459981097ee99605c57fec95c637cf59aaa20fe766 +c1 v3 frozen-packages/668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf 668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf 0e23f7747cc63014bade7543efaf745e7e9a7e5d6dee2a48c602ef7a3eba091e diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/launch-schedule.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/launch-schedule.tsv new file mode 100644 index 0000000000..57b78f4de9 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/launch-schedule.tsv @@ -0,0 +1,81 @@ +run_id cell_id wave target_label condition_label replicate +r001 0fea6fbdfedb1e8f9c1c331f5a47fa16 1 m7 c0 1 +r002 4886a308ee81e3c948ea133df9661771 1 m3 c1 1 +r003 fd82077e8d40fcaf1f5a51da67a9bc70 1 m0 c0 1 +r004 a01a89ac57f92484e62f719916fabcde 1 m6 c0 1 +r005 c51d935dd66af61d6ae7370471dd270b 1 m1 c0 1 +r006 237314e935a671dda9f0b729a9164e42 1 m4 c0 1 +r007 2d557eebd147400908963542447f78d8 1 m2 c1 1 +r008 b7d49044025495a68243c59ea223d392 1 m7 c1 1 +r009 4a591e9e7f2b1706efec55c9b809b15e 1 m5 c1 1 +r010 b4c7cb4a6c9a884747af60df5bbe135e 1 m6 c1 1 +r011 df3cf01648593724319ee30d40e53694 1 m2 c0 1 +r012 b9449281466536eb625d875c9a1b52c6 1 m5 c0 1 +r013 c08c9cbec10b09416dedb07222a8fa89 1 m0 c1 1 +r014 225c7b447026b9d4fb60d0953329f336 1 m3 c0 1 +r015 4dfb61f35f3ba9347c43f9c7390941ba 1 m4 c1 1 +r016 0ddb17b601260a6fa49fd03e146c8bf4 1 m1 c1 1 +r017 c0867a2b7f8ca30ce79eaf94a8c2813b 2 m1 c0 3 +r018 ef1a0677a6ed30a4b2ccf4bf2c4fa075 2 m4 c1 3 +r019 07041e4c7851c15fbd8ca3b5b89a649c 2 m0 c1 3 +r020 0a93b4a0423042598341734b5b80aea2 2 m1 c1 3 +r021 45d746368e63763aae22c259c646af5e 2 m0 c0 3 +r022 cf391d64b44517b85ca71858bb6d8e52 2 m4 c0 3 +r023 136b2c4efefe0bab1417f14b38a57793 2 m6 c1 3 +r024 aea81024b16085dc0bc078cab31e9b6d 2 m5 c1 3 +r025 83498ad4b1212eaea7b615f0292a1ce3 2 m3 c0 3 +r026 b9a35f06e08e96ce9b0b093da1de80da 2 m7 c0 3 +r027 a29afd5bf972ec132f78543ba750e93f 2 m3 c1 3 +r028 8beb0474c46f57c79e68aa2f12dfd6a6 2 m2 c0 3 +r029 a3f969c74a0d22bd838ad11dd92aa415 2 m2 c1 3 +r030 919322be3592430d06791a08b1aa8cea 2 m6 c0 3 +r031 c42948b54a7eab85fd103b78c835c7ac 2 m7 c1 3 +r032 418a4bcde6dda1a6187bae8547d85a93 2 m5 c0 3 +r033 ca8819a9835763564d3e5c7b365393d8 3 m1 c0 5 +r034 1ff38d28876c4cdd6329bfeaa1385e6f 3 m4 c1 5 +r035 07fde893dca23f5e7b0b0e576c212375 3 m3 c0 5 +r036 bebc72c4981368d1bffe7e85d5e68357 3 m3 c1 5 +r037 eef95e01264a728a7c6eabe725507b5a 3 m0 c0 5 +r038 628efb38e96428481f2598bd5207472d 3 m2 c1 5 +r039 ba7abe98e80c53f75aa8f8238c7ffe03 3 m1 c1 5 +r040 de3c340e44bf752186e73da5cdbc05bd 3 m2 c0 5 +r041 1ec17f7769c0078f9fcbed12eb0be0f7 3 m5 c0 5 +r042 5edc22ffe7b0779af2a68e40283db476 3 m4 c0 5 +r043 9fed5500b2156c7c3cb16aff89a5a89f 3 m6 c0 5 +r044 32e9f759f85e81477aeb318b0e0497c2 3 m5 c1 5 +r045 434b916b4f0630b6e6aa447c4048f8d6 3 m7 c0 5 +r046 8a37e9f35d221302948bb0526f432b98 3 m6 c1 5 +r047 86e15f99caa5c1efc10a11768832390e 3 m0 c1 5 +r048 aeea1b0c5420db0d4c06d58b2708366e 3 m7 c1 5 +r049 16cb821ea568942a5847dc01ce421b41 4 m4 c0 2 +r050 ada6c44a0f72bc924d6b9f094cad0ff0 4 m5 c0 2 +r051 d1bfa4935ce8286684864898b004f962 4 m6 c0 2 +r052 f0309a354f74b08a1a0972b62fac6ccd 4 m2 c1 2 +r053 296be450c238efd67a62d38623894010 4 m5 c1 2 +r054 7da22994e57951eb951e0e387d852650 4 m3 c1 2 +r055 4612a4154bf19ccc48a84d507fe9b05b 4 m2 c0 2 +r056 e6632dafee22c0cfb734d7911d78769f 4 m7 c0 2 +r057 1ab98c2830b91afc592e74e6e6c5b9be 4 m0 c1 2 +r058 bf0abeb5157d3eed578e8bc0edde0d47 4 m1 c0 2 +r059 91104a670ad522d0fe606ac8e15dc1f5 4 m3 c0 2 +r060 02038086e3c763683e357ad7d23a8005 4 m7 c1 2 +r061 b0f33b6db364346c2523b34bcc8a0ecd 4 m1 c1 2 +r062 359c5fe574e54bea738b01822c7104f6 4 m6 c1 2 +r063 812bd420e5eba308dc68e07047e583c1 4 m0 c0 2 +r064 5f63b439ef17bd4c9938d31e4445e847 4 m4 c1 2 +r065 2505caff3f06d70966adcc385f251627 5 m7 c1 4 +r066 da357b62c59f84a5e9f0f0a11214900a 5 m4 c1 4 +r067 ae051f5093ea311894002930e26944f6 5 m0 c1 4 +r068 6a4a5f007b65de786b2391d09453d5e7 5 m7 c0 4 +r069 40e593bb6d93cf7b49e1fb35342ff99a 5 m2 c0 4 +r070 1769a58b00a1db740e39dbd7cba09c6e 5 m5 c0 4 +r071 4cec2104444a8240d06ee311657dc0f3 5 m0 c0 4 +r072 843cbdc69d6a93ba2d97c2ef54f74264 5 m2 c1 4 +r073 59206a03a32ed8ca983f8665c26fcacc 5 m1 c0 4 +r074 6367c11d5940c2dbeeb37e422d2b79c8 5 m3 c0 4 +r075 02d88d35c49cd6779edc8578e7d89bdb 5 m3 c1 4 +r076 ecf25044b6f65e28e446370bea636c89 5 m5 c1 4 +r077 886d6d32ed0e40aa1f82f3f94cfcbcaf 5 m6 c1 4 +r078 2a9b5392e246efa52195b90613899322 5 m4 c0 4 +r079 c8dd0a199fe8db9173fe43372c47e05b 5 m6 c0 4 +r080 aaf0db751da7393ff60ded2b6d179fb7 5 m1 c1 4 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/presentation-orders.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/presentation-orders.tsv new file mode 100644 index 0000000000..744fed7552 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/presentation-orders.tsv @@ -0,0 +1,17 @@ +claim labels_in_order +S-s1 A,B,E,F,G,I,J,C,D,H +S-s2 J,E,F,I,B,G,H,C,D,A +C-s1 H,F,J,C,B,A,I,G,E,D +C-s2 I,J,G,H,F,D,A,C,E,B +X-s1 C,F,I,B,A,H,G,D,J,E +X-s2 H,D,B,C,F,E,A,J,G,I +Q-s1 D,B,E,F,A,I,G,J,H,C +Q-s2 C,B,D,A,G,J,I,F,H,E +W-s1 H,F,B,E,C,A,I,D,J,G +W-s2 F,G,E,C,A,B,H,J,D,I +M-s1 F,E,D,G,A,C,J,I,H,B +M-s2 G,J,A,B,F,E,H,C,I,D +R-s1 H,F,G,I,A,D,J,E,C,B +R-s2 A,J,G,B,F,I,C,E,D,H +K-s1 H,D,J,I,F,G,A,C,B,E +K-s2 J,F,H,I,A,B,E,D,C,G diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/scoring-schedule.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/scoring-schedule.tsv new file mode 100644 index 0000000000..40a5798be0 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/scoring-schedule.tsv @@ -0,0 +1,17 @@ +claim +S-s1 +M-s1 +Q-s2 +M-s2 +W-s2 +R-s2 +W-s1 +Q-s1 +S-s2 +X-s1 +X-s2 +R-s1 +C-s1 +C-s2 +K-s1 +K-s2 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/seeds.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/seeds.json new file mode 100644 index 0000000000..173d339364 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/seeds.json @@ -0,0 +1,7 @@ +{ + "condition": "cd806c526988b02b7ec4c4cb13a20f4bddb2e175787e20044c65c87e7dba5b8e", + "schedule": "bacd8b99a28e6368f7d0fb8996bf5fff0546407c552e6315135ea6509ef4ac79", + "blind": "270254466233a651d3e442a7a5394cf40db13d824b84cd702a75501960815522", + "presentation": "893456b29122ef6e27acb3a82f3f5c08cb266c1a7b246a98eaaa072f6de1ee0a", + "scorer": "ea4a345be348a08359dbd6994b92b83cc3241d714fa165a5f8254462372ad755" +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/target-map.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/target-map.tsv new file mode 100644 index 0000000000..7b6e4fbfbb --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/target-map.tsv @@ -0,0 +1,9 @@ +target_label mode source_path tree_sha256 word_cap +m0 M fixtures/v3-targeted/m_multirelease b269cf068196d1c06b87be6bcded494827e7ac8a2cb6debf1eb0f0f5d0388479 1800 +m1 K fixtures/v3-targeted/k_regression ca272e524b36a892e25f6169631a184ff764026451eff2f6ac4ab8e9d5e87ea2 2200 +m2 R fixtures/v3-targeted/r_redesign 6d1a41909b012484d71f94d194071a7ebbb6773b7596db88127ca2a195b70ffc 1800 +m3 S fixtures/v3-targeted/s_symbolic 28ecc523e15b914a187814ab2752c0d85996948a552c62f970d8484bc6ed467a 1800 +m4 X fixtures/v3-targeted/x_cross 25b4efef689601f3b5983bf6b914bd367153d0b21a6f1f4d40de50c5d412afa7 2400 +m5 W fixtures/v3-targeted/w_whole_execution b27b95fbc9ffa9d335bb6b4614a9f227a5798122fca79b42cf53c9b106a6aff6 1800 +m6 C fixtures/v3-targeted/c_conflict 065c3cfc032af93e7576e17e49322826c4a379a707870175b16c2663d1e8e4e0 1800 +m7 Q fixtures/v3-targeted/q_quantifiers c0a4c43373a159cb38d08724af8b02187b249ab6a73f7e1b10b2276f38b0cb5a 1800