From 6880d2b4691a8f3e722ffd0758395eedfc7beef0 Mon Sep 17 00:00:00 2001 From: Josh Liebow-Feeser Date: Sat, 1 Aug 2026 03:20:41 +0000 Subject: [PATCH] Freeze the V3 targeted confirmation protocol Freeze a blind 80-report evaluation of V3 against V2: eight modes, two conditions, and five fresh replicates per cell. Seal condition maps, prompts, fixtures, frozen skill packages, rubrics, allowed authority, expected atoms, and report and scoring schemas before generation begins. Retain mode-level analysis for symbolic release domains, nonlinear policy composition, configuration products, existential unsoundness certificates, whole-execution behavioral claims, positive multi-version proofs, abstraction-design firewalls, and regression breadth. Require candidate V3 to pass every required atom in every replicate and produce zero hard errors, TCB or authority defects, proposal laundering, semantic noncompletion, and scope or budget failures. Add an append-only event ledger and explicit attempt lifecycle so generation, validation, retries, blind scoring, adjudication, and unblinding remain auditable. Record the independent package and protocol reviews. At freeze time the report count is zero, so later results cannot have influenced the question or success criteria. gherrit-pr-id: G5k3oylk4nmllritz5hdpiksvqbp24ffs --- .../runs/2026-07-31-v3-targeted/events.jsonl | 1 + .../2026-07-31-v3-targeted/fetch_authority.py | 219 + .../2026-07-31-v3-targeted/freeze/LOCK.json | 23 + .../freeze/allowlists/C.txt | 18 + .../freeze/allowlists/K.txt | 8 + .../freeze/allowlists/M.txt | 11 + .../freeze/allowlists/Q.txt | 1 + .../freeze/allowlists/R.txt | 2 + .../freeze/allowlists/S.txt | 12 + .../freeze/allowlists/W.txt | 2 + .../freeze/allowlists/X.txt | 22 + .../freeze/authority-manifest.tsv | 77 + .../freeze/file-manifest.sha256 | 47 + .../freeze/manifest.json | 101 + .../freeze/oracle}/controls.md | 43 +- .../freeze/oracle}/domain.md | 12 +- .../freeze/oracle}/verdict.md | 20 +- .../2026-07-31-v3-targeted/freeze/plan.md} | 50 +- .../freeze/policies/isolation.md | 44 + .../freeze/policies/reruns.md | 43 + .../freeze/policies/tools.md | 16 + .../freeze/policies/word-count.md | 22 + .../freeze/prompts/adjudicator.md | 23 + .../freeze/prompts/report.md | 36 + .../freeze/prompts/scorer.md | 28 + .../freeze/randomization/commitments.json | 11 + .../freeze/randomization/spec.md | 41 + .../freeze/rubrics/C.md | 128 + .../freeze/rubrics/K.md | 97 + .../freeze/rubrics/M.md | 171 + .../freeze/rubrics/Q.md | 76 + .../freeze/rubrics/R.md | 67 + .../freeze/rubrics/S.md | 111 + .../freeze/rubrics/SCORER.md | 72 + .../freeze/rubrics/W.md | 66 + .../freeze/rubrics/X.md | 145 + .../freeze/schemas/adjudication.schema.json | 35 + .../freeze/schemas/disagreements.schema.json | 36 + .../freeze/schemas/event.schema.json | 27 + .../freeze/schemas/final.schema.json | 33 + .../freeze/schemas/score.schema.json | 91 + .../2026-07-31-v3-targeted/operations.lock | 0 .../runs/2026-07-31-v3-targeted/prepare.py | 240 ++ .../runs/2026-07-31-v3-targeted/protocol.py | 3603 +++++++++++++++++ .../sealed/blind-map.tsv | 81 + .../sealed/condition-map.tsv | 3 + .../sealed/launch-schedule.tsv | 81 + .../sealed/presentation-orders.tsv | 17 + .../sealed/scoring-schedule.tsv | 17 + .../2026-07-31-v3-targeted/sealed/seeds.json | 7 + .../sealed/target-map.tsv | 9 + 51 files changed, 6104 insertions(+), 42 deletions(-) create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/events.jsonl create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/fetch_authority.py create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/LOCK.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/C.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/K.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/M.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/Q.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/R.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/S.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/W.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/X.txt create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/authority-manifest.tsv create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/file-manifest.sha256 create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/manifest.json rename evals/unsafe-rust/{v3-targeted-drafts => runs/2026-07-31-v3-targeted/freeze/oracle}/controls.md (85%) rename evals/unsafe-rust/{v3-targeted-drafts => runs/2026-07-31-v3-targeted/freeze/oracle}/domain.md (99%) rename evals/unsafe-rust/{v3-targeted-drafts => runs/2026-07-31-v3-targeted/freeze/oracle}/verdict.md (96%) rename evals/unsafe-rust/{v3-targeted-plan.md => runs/2026-07-31-v3-targeted/freeze/plan.md} (81%) create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/isolation.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/reruns.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/tools.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/word-count.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/adjudicator.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/report.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/scorer.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/commitments.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/spec.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/C.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/K.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/M.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/Q.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/R.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/S.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/SCORER.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/W.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/X.md create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/adjudication.schema.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/disagreements.schema.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/event.schema.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/final.schema.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/score.schema.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/operations.lock create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/prepare.py create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/protocol.py create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/blind-map.tsv create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/condition-map.tsv create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/launch-schedule.tsv create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/presentation-orders.tsv create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/scoring-schedule.tsv create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/seeds.json create mode 100644 evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/target-map.tsv diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/events.jsonl b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/events.jsonl new file mode 100644 index 0000000000..5956a40e03 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/events.jsonl @@ -0,0 +1 @@ +{"details": {"lock_sha256": "9aeeaa4987227f5ac4761349d0979ef6b425cb724fb54d1fe21b89f9cb62e678"}, "event": "freeze_locked", "phase": "freeze", "previous_event_sha256": null, "schema_version": 1, "sequence": 1, "sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", "time_utc": "2026-08-01T03:19:38.915424Z"} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/fetch_authority.py b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/fetch_authority.py new file mode 100644 index 0000000000..121d368499 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/fetch_authority.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Fetch or verify the exact official-document pages allowed by this run.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import html +import io +import re +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + + +RUN = Path(__file__).resolve().parent +ALLOWLISTS = RUN / "freeze" / "allowlists" +MANIFEST = RUN / "freeze" / "authority-manifest.tsv" +MODES = ("S", "C", "X", "Q", "W", "M", "R", "K") +FIELDS = ( + "mode", + "requested_url", + "fetch_url", + "final_url", + "status", + "content_type", + "bytes", + "sha256", + "fragment_found", + "retrieved_utc", +) + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def read_pinned(path: Path) -> bytes: + data = path.read_bytes() + if (RUN / "freeze" / "LOCK.json").exists(): + import protocol + + if sha256(data) != protocol.frozen_file_digest(path): + raise ValueError(f"frozen authority input changed while being read: {path}") + return data + + +def load_urls() -> list[tuple[str, str]]: + rows: list[tuple[str, str]] = [] + for mode in MODES: + path = ALLOWLISTS / f"{mode}.txt" + for url in read_pinned(path).decode().splitlines(): + if not re.fullmatch(r"https://doc\.rust-lang\.org/\S+", url): + raise ValueError(f"invalid allowlisted URL: {url!r}") + rows.append((mode, url)) + return rows + + +def without_fragment(url: str) -> str: + parts = urllib.parse.urlsplit(url) + return urllib.parse.urlunsplit((parts.scheme, parts.netloc, parts.path, parts.query, "")) + + +def fragment_present(requested_url: str, body: bytes) -> bool: + fragment = urllib.parse.unquote(urllib.parse.urlsplit(requested_url).fragment) + if not fragment: + return True + text = body.decode("utf-8", errors="replace") + escaped = html.escape(fragment, quote=True) + patterns = ( + rf'\bid=["\']{re.escape(fragment)}["\']', + rf'\bid=["\']{re.escape(escaped)}["\']', + rf'\bname=["\']{re.escape(fragment)}["\']', + ) + return any(re.search(pattern, text) for pattern in patterns) + + +def fetch(url: str) -> dict[str, object]: + request = urllib.request.Request( + url, + headers={"User-Agent": "unsafe-rust-skill-evaluation/1 authority-freeze"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read() + return { + "final_url": response.geturl(), + "status": response.status, + "content_type": response.headers.get_content_type(), + "bytes": len(body), + "sha256": sha256(body), + "body": body, + "retrieved_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + } + + +def build_rows() -> list[dict[str, str]]: + requested = load_urls() + fetched: dict[str, dict[str, object]] = {} + for _mode, requested_url in requested: + fetch_url = without_fragment(requested_url) + if fetch_url not in fetched: + fetched[fetch_url] = fetch(fetch_url) + + rows: list[dict[str, str]] = [] + for mode, requested_url in requested: + fetch_url = without_fragment(requested_url) + result = fetched[fetch_url] + found = fragment_present(requested_url, result["body"]) + if not found: + raise ValueError(f"fragment not present in retrieved page: {requested_url}") + rows.append( + { + "mode": mode, + "requested_url": requested_url, + "fetch_url": fetch_url, + "final_url": str(result["final_url"]), + "status": str(result["status"]), + "content_type": str(result["content_type"]), + "bytes": str(result["bytes"]), + "sha256": str(result["sha256"]), + "fragment_found": "true", + "retrieved_utc": str(result["retrieved_utc"]), + } + ) + return rows + + +def render(rows: list[dict[str, str]]) -> str: + output = io.StringIO(newline="") + writer = csv.DictWriter(output, fieldnames=FIELDS, dialect="excel-tab", lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + return output.getvalue() + + +def read_manifest() -> list[dict[str, str]]: + rows = list( + csv.DictReader(io.StringIO(read_pinned(MANIFEST).decode()), dialect="excel-tab") + ) + if not rows or tuple(rows[0]) != FIELDS: + raise ValueError("authority manifest has unexpected columns") + return rows + + +def verify() -> None: + rows = read_manifest() + expected_pairs = load_urls() + actual_pairs = [(row["mode"], row["requested_url"]) for row in rows] + if actual_pairs != expected_pairs: + raise ValueError("authority manifest does not match allowlist ordering") + + fetched: dict[str, dict[str, object]] = {} + for row in rows: + fetch_url = row["fetch_url"] + if fetch_url != without_fragment(row["requested_url"]): + raise ValueError(f"incorrect fetch URL for {row['requested_url']}") + if fetch_url not in fetched: + fetched[fetch_url] = fetch(fetch_url) + result = fetched[fetch_url] + checks = { + "final_url": str(result["final_url"]), + "status": str(result["status"]), + "content_type": str(result["content_type"]), + "bytes": str(result["bytes"]), + "sha256": str(result["sha256"]), + } + for field, actual in checks.items(): + if row[field] != actual: + raise ValueError( + f"authority drift for {row['requested_url']}: " + f"{field} frozen={row[field]!r} live={actual!r}" + ) + if row["fragment_found"] != "true" or not fragment_present( + row["requested_url"], result["body"] + ): + raise ValueError(f"fragment missing for {row['requested_url']}") + print(f"verified {len(rows)} allowlist entries across {len(fetched)} pages") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--write", action="store_true", help="retrieve and write the frozen manifest") + parser.add_argument( + "--record-wave", + type=int, + choices=range(1, 6), + help="verify live bytes and append a successful verification event for this collection wave", + ) + args = parser.parse_args() + if args.write and args.record_wave is not None: + raise SystemExit("--write and --record-wave are mutually exclusive") + if args.write: + if (RUN / "freeze" / "LOCK.json").exists(): + raise SystemExit("refusing to rewrite authority manifest after freeze lock") + MANIFEST.write_text(render(build_rows())) + print(f"wrote {MANIFEST.relative_to(RUN)}") + else: + operation_lock_handle = None + if args.record_wave is not None: + import protocol + + operation_lock_handle = protocol.acquire_operation_lock() + protocol.validate_static(require_lock=True, announce=False) + protocol.assert_freeze_locked() + protocol.assert_authority_verification_allowed(args.record_wave) + verify() + if args.record_wave is not None: + protocol.append_event( + "collection", + "authority_verified", + digest=protocol.frozen_file_digest(MANIFEST), + details={"wave": args.record_wave, "entries": len(read_manifest())}, + ) + + +if __name__ == "__main__": + main() diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/LOCK.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/LOCK.json new file mode 100644 index 0000000000..08c149717a --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/LOCK.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "status": "FROZEN", + "file_manifest_sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", + "review_signoffs": [ + { + "reviewer_id": "codex-analyze-n-overall", + "verdict": "PASS/FREEZE", + "file_manifest_sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", + "scope": "Independent authentication and holistic review of all 47 manifest-listed bytes: protocol.py, prepare.py, fetch_authority.py, frozen plan/manifest/policies/prompts/schemas/randomization/allowlists/authority manifest/oracles/rubrics, sealed maps/seeds, V3 and V2 package identities, all eight target identities and source-oracle claims, and zero-report precollection state.", + "reviewed_utc": "2026-08-01T03:16:47Z" + }, + { + "reviewer_id": "v3-fixtures-domain-audit", + "verdict": "PASS/FREEZE", + "file_manifest_sha256": "2a036052cd3def40582bbe6c3fa7f81ff0bb4afbc82b86f249885076ae7974b3", + "scope": "Independent byte authentication of all 47 manifest-listed files; complete review of protocol lifecycle, freeze and failure semantics, policies, prompts, schemas, randomization and sealed maps, blinding, authority allowlists and oracle/rubric semantics, V3 and V2 package identities, all eight fixture and target identities, and zero-report precollection state. The 76 allowlist entries across 46 live official pages were also reverified against frozen hashes without recording an event.", + "reviewed_utc": "2026-08-01T03:18:56Z" + } + ], + "reports_collected_before_lock": 0, + "locked_utc": "2026-08-01T03:19:11Z" +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/C.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/C.txt new file mode 100644 index 0000000000..b65efda7ef --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/C.txt @@ -0,0 +1,18 @@ +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_or +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_or +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_or +https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html +https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html +https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/K.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/K.txt new file mode 100644 index 0000000000..5c46faa1e1 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/K.txt @@ -0,0 +1,8 @@ +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.len +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.is_empty +https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html#arithmetic-and-logical-binary-operators +https://doc.rust-lang.org/1.82.0/reference/types/numeric.html#integer-types +https://doc.rust-lang.org/1.82.0/reference/items/traits.html#unsafe-traits +https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html#unsafe-traits-unsafe-trait diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/M.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/M.txt new file mode 100644 index 0000000000..d74c62056e --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/M.txt @@ -0,0 +1,11 @@ +https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html +https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html +https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html +https://doc.rust-lang.org/1.80.0/reference/type-layout.html#primitive-data-layout +https://doc.rust-lang.org/1.80.0/std/primitive.u8.html#impl-Copy-for-u8 +https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html +https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html +https://doc.rust-lang.org/1.80.0/std/primitive.u32.html#impl-Copy-for-u32 +https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html +https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#impl-Copy-for-u32 +https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/Q.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/Q.txt new file mode 100644 index 0000000000..d467a88cf6 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/Q.txt @@ -0,0 +1 @@ +https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/R.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/R.txt new file mode 100644 index 0000000000..7515068ad3 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/R.txt @@ -0,0 +1,2 @@ +https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut +https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#method.wrapping_add diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/S.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/S.txt new file mode 100644 index 0000000000..f13c981a34 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/S.txt @@ -0,0 +1,12 @@ +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none +https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked +https://doc.rust-lang.org/1.86.0/releases.html#version-1841-2025-01-30 +https://doc.rust-lang.org/1.86.0/releases.html#version-1851-2025-03-18 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/W.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/W.txt new file mode 100644 index 0000000000..655f04c58c --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/W.txt @@ -0,0 +1,2 @@ +https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety +https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/X.txt b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/X.txt new file mode 100644 index 0000000000..25e171dfb3 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/allowlists/X.txt @@ -0,0 +1,22 @@ +https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html#method.new_unchecked +https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html#comparison-operators +https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html +https://doc.rust-lang.org/1.85.1/std/macro.panic.html +https://doc.rust-lang.org/1.85.1/std/env/fn.var.html +https://doc.rust-lang.org/1.85.1/std/string/struct.String.html#method.as_str +https://doc.rust-lang.org/1.85.1/std/primitive.str.html#impl-ToOwned-for-str +https://doc.rust-lang.org/1.85.1/std/fmt/index.html#syntax +https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html +https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html +https://doc.rust-lang.org/1.85.1/reference/patterns.html#literal-patterns +https://doc.rust-lang.org/1.85.1/reference/patterns.html#or-patterns +https://doc.rust-lang.org/1.85.1/reference/patterns.html#wildcard-pattern +https://doc.rust-lang.org/1.85.1/std/macro.println.html +https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#conditional-compilation +https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#the-cfg-attribute +https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-cfg +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-check-cfg +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rerun-if-env-changed +https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#life-cycle-of-a-build-script +https://doc.rust-lang.org/1.85.1/cargo/reference/features.html diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/authority-manifest.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/authority-manifest.tsv new file mode 100644 index 0000000000..bead2c56bb --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/authority-manifest.tsv @@ -0,0 +1,77 @@ +mode requested_url fetch_url final_url status content_type bytes sha256 fragment_found retrieved_utc +S https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +S https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +S https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html 200 text/html 213924 4e91aab56b50dbf245e6a1ea950ad74f4cb8971de958a7d4658d964a0a6350f5 true 2026-08-01T01:28:38.445793Z +S https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html https://doc.rust-lang.org/1.84.1/std/option/enum.Option.html 200 text/html 213924 4e91aab56b50dbf245e6a1ea950ad74f4cb8971de958a7d4658d964a0a6350f5 true 2026-08-01T01:28:38.445793Z +S https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +S https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +S https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html 200 text/html 214310 3e342d6b7b88a60eef09f8452d071040bf2e281e7d8228aaf727740be4015a15 true 2026-08-01T01:28:38.575213Z +S https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html https://doc.rust-lang.org/1.85.1/std/option/enum.Option.html 200 text/html 214310 3e342d6b7b88a60eef09f8452d071040bf2e281e7d8228aaf727740be4015a15 true 2026-08-01T01:28:38.575213Z +S https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +S https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +S https://doc.rust-lang.org/1.86.0/releases.html#version-1841-2025-01-30 https://doc.rust-lang.org/1.86.0/releases.html https://doc.rust-lang.org/1.86.0/releases.html 200 text/html 993212 5a25142ffcdc8a32d27623a01bde7e05f5dda96c172f87aa5d2b30e465c28998 true 2026-08-01T01:28:38.732157Z +S https://doc.rust-lang.org/1.86.0/releases.html#version-1851-2025-03-18 https://doc.rust-lang.org/1.86.0/releases.html https://doc.rust-lang.org/1.86.0/releases.html 200 text/html 993212 5a25142ffcdc8a32d27623a01bde7e05f5dda96c172f87aa5d2b30e465c28998 true 2026-08-01T01:28:38.732157Z +C https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +C https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +C https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html#method.unwrap_or https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html https://doc.rust-lang.org/1.84.0/std/option/enum.Option.html 200 text/html 213924 a395b4f6dfe1fc5242b61a13ef3c0109033f8053b3fb0772b5eb74463ca69105 true 2026-08-01T01:28:38.381052Z +C https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +C https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +C https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html#method.unwrap_or https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html https://doc.rust-lang.org/1.85.0/std/option/enum.Option.html 200 text/html 214310 e4af7fc387fab0fd0fbc29db6442f41c7a35258d03b3b4e7fa645190f7b8c86e true 2026-08-01T01:28:38.510278Z +C https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.is_none https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +C https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_unchecked https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +C https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html#method.unwrap_or https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html https://doc.rust-lang.org/1.86.0/std/option/enum.Option.html 200 text/html 214377 733fe2d8a415699250fdba975b9b943c02c5fe27e2dfe0822eb1d20f522864ea true 2026-08-01T01:28:38.642322Z +C https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html 200 text/html 36314 c231b94b17857c57b47336ebede1e3e81ee8d58db68e669a6b1da1026164ac3b true 2026-08-01T01:28:38.772524Z +C https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html 200 text/html 36314 c231b94b17857c57b47336ebede1e3e81ee8d58db68e669a6b1da1026164ac3b true 2026-08-01T01:28:38.772524Z +C https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html 200 text/html 6436 b92f2f9939bf888edd27d156eea4f5a48a98867752198001dc6b12253973fedd true 2026-08-01T01:28:38.811777Z +C https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:38.854337Z +C https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:38.854337Z +C https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html 200 text/html 6436 2a9462a947c44b60b1131aa5f1378b8ebbb47df234aa152232d935f5a95bc8bc true 2026-08-01T01:28:38.891018Z +C https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html 200 text/html 39866 f5ec1c4381d47fa28bbfe7b92f0c4383c0f8a30bc7121831fa4a4cd4a5543f4c true 2026-08-01T01:28:38.934329Z +C https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html 200 text/html 39866 f5ec1c4381d47fa28bbfe7b92f0c4383c0f8a30bc7121831fa4a4cd4a5543f4c true 2026-08-01T01:28:38.934329Z +C https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html 200 text/html 6503 0aa2780471d5ac4b0b5e746ea7aa01eb78fbabf9cd584a96d3088ec1afcc5883 true 2026-08-01T01:28:38.971684Z +X https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html#method.new_unchecked https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html https://doc.rust-lang.org/1.85.1/std/num/struct.NonZero.html 200 text/html 1396156 c1393af3fca7ba25fd448504d6f70fc01fea71efac9cc50052178899ccb877dc true 2026-08-01T01:28:39.066223Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html#comparison-operators https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/operator-expr.html 200 text/html 58441 da30ee41874a27f0d5d991a9f53dda6d4b59e46e8f28ad06c0c714b941a9d3e0 true 2026-08-01T01:28:39.115780Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/if-expr.html 200 text/html 16334 66cb23b4db83562de51a0b3411d473b430c738aa934c44b75f30aa2dd02c8f05 true 2026-08-01T01:28:39.154617Z +X https://doc.rust-lang.org/1.85.1/std/macro.panic.html https://doc.rust-lang.org/1.85.1/std/macro.panic.html https://doc.rust-lang.org/1.85.1/std/macro.panic.html 200 text/html 10472 c7f3fe8e00889d1d9206fa39a373cfd060b993c5377b169979cc947e9efcecc0 true 2026-08-01T01:28:39.192926Z +X https://doc.rust-lang.org/1.85.1/std/env/fn.var.html https://doc.rust-lang.org/1.85.1/std/env/fn.var.html https://doc.rust-lang.org/1.85.1/std/env/fn.var.html 200 text/html 5859 4254c02ef260e2a7729980cc6f84796a5b3770ac58334a598fc0b745df2d805f true 2026-08-01T01:28:39.231253Z +X https://doc.rust-lang.org/1.85.1/std/string/struct.String.html#method.as_str https://doc.rust-lang.org/1.85.1/std/string/struct.String.html https://doc.rust-lang.org/1.85.1/std/string/struct.String.html 200 text/html 577036 d94dac44c1c9d360c7caf0806c0c41b4d2330913463d0be42eeb211416563006 true 2026-08-01T01:28:39.310266Z +X https://doc.rust-lang.org/1.85.1/std/primitive.str.html#impl-ToOwned-for-str https://doc.rust-lang.org/1.85.1/std/primitive.str.html https://doc.rust-lang.org/1.85.1/std/primitive.str.html 200 text/html 563520 f6569e4e89b98037dbba1106d5e1c7d0598d6592c48eaa6a42779e72430ca988 true 2026-08-01T01:28:39.395145Z +X https://doc.rust-lang.org/1.85.1/std/fmt/index.html#syntax https://doc.rust-lang.org/1.85.1/std/fmt/index.html https://doc.rust-lang.org/1.85.1/std/fmt/index.html 200 text/html 65822 3d3493d16ea89a7626d9f5036456924d617de6d50002b2c0ed4a55ba36be53a7 true 2026-08-01T01:28:39.444849Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/block-expr.html 200 text/html 23305 eca5d1df45aaac303bdaa0a7a0a7e93f2e48b38171325e963bed90d3dbaf31ba true 2026-08-01T01:28:39.485645Z +X https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html https://doc.rust-lang.org/1.85.1/reference/expressions/match-expr.html 200 text/html 17731 281bbc7dfe2c0f330e7229684b55747cd3392845fad3a5be536ec5a4918ed695 true 2026-08-01T01:28:39.560464Z +X https://doc.rust-lang.org/1.85.1/reference/patterns.html#literal-patterns https://doc.rust-lang.org/1.85.1/reference/patterns.html https://doc.rust-lang.org/1.85.1/reference/patterns.html 200 text/html 58174 4e3e6090c0a65f5d26e7ae807f4241f35b91e8fbd24a76ecb23a07bf4fd6d0cf true 2026-08-01T01:28:39.614415Z +X https://doc.rust-lang.org/1.85.1/reference/patterns.html#or-patterns https://doc.rust-lang.org/1.85.1/reference/patterns.html https://doc.rust-lang.org/1.85.1/reference/patterns.html 200 text/html 58174 4e3e6090c0a65f5d26e7ae807f4241f35b91e8fbd24a76ecb23a07bf4fd6d0cf true 2026-08-01T01:28:39.614415Z +X https://doc.rust-lang.org/1.85.1/reference/patterns.html#wildcard-pattern https://doc.rust-lang.org/1.85.1/reference/patterns.html https://doc.rust-lang.org/1.85.1/reference/patterns.html 200 text/html 58174 4e3e6090c0a65f5d26e7ae807f4241f35b91e8fbd24a76ecb23a07bf4fd6d0cf true 2026-08-01T01:28:39.614415Z +X https://doc.rust-lang.org/1.85.1/std/macro.println.html https://doc.rust-lang.org/1.85.1/std/macro.println.html https://doc.rust-lang.org/1.85.1/std/macro.println.html 200 text/html 6697 05b1c308b73cceceb09edd797b1cfb51b88039c26c9cd018aeda106fad2aa076 true 2026-08-01T01:28:39.653511Z +X https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#conditional-compilation https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:39.693555Z +X https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#the-cfg-attribute https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html 200 text/html 36314 083c3862cabe9c0b67cb63a12e254376b7769e8826d0864b32ab9fc8a737f31f true 2026-08-01T01:28:39.693555Z +X https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html 200 text/html 6436 c9f3ba7f85fed765c6485ac0754891ee29b9f68efee68d60967a547264ffb748 true 2026-08-01T01:28:39.731921Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-cfg https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rustc-check-cfg https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#rerun-if-env-changed https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html#life-cycle-of-a-build-script https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html https://doc.rust-lang.org/1.85.1/cargo/reference/build-scripts.html 200 text/html 40211 1247cbaf8ce775f17349367d13ac4eecc6d9cfa343310f12d8c1deccd19e07b2 true 2026-08-01T01:28:39.779201Z +X https://doc.rust-lang.org/1.85.1/cargo/reference/features.html https://doc.rust-lang.org/1.85.1/cargo/reference/features.html https://doc.rust-lang.org/1.85.1/cargo/reference/features.html 200 text/html 36803 96b2337cd60180df5a8566f343e52938dfcaa369bf12e1e82723a2326f64cb25 true 2026-08-01T01:28:39.818858Z +Q https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html 200 text/html 11603 555597c0db28f65466dd734a9f57d4aaca8abe7f6e0e256b3f0d64a877529fd3 true 2026-08-01T01:28:39.858491Z +W https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html 200 text/html 11603 555597c0db28f65466dd734a9f57d4aaca8abe7f6e0e256b3f0d64a877529fd3 true 2026-08-01T01:28:39.858491Z +W https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html 200 text/html 37852 9f6deb0ebbdddd0362a3406ccb872c84108034c55c7d5b3e124d50b5d2cae9a9 true 2026-08-01T01:28:39.900179Z +M https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html 200 text/html 9404 f31ae6a889da541592c796e521b7abb18fa4b7a4dc25da32e839d076fe8aeba3 true 2026-08-01T01:28:39.944111Z +M https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html 200 text/html 9404 3c9e77c0d03f9669ba5342cff73a408360e0cc6bda1bc17a368234cf77f5ae4c true 2026-08-01T01:28:39.984139Z +M https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html 200 text/html 10246 c708643d37fa87df877526f34c6bc3021e0fd17c062e29b854da58b0b98843b4 true 2026-08-01T01:28:40.022195Z +M https://doc.rust-lang.org/1.80.0/reference/type-layout.html#primitive-data-layout https://doc.rust-lang.org/1.80.0/reference/type-layout.html https://doc.rust-lang.org/1.80.0/reference/type-layout.html 200 text/html 55218 f1e8382edb288ae23f8a2e654910addf3540f48f635bec80f0242f3c58b4b78c true 2026-08-01T01:28:40.070245Z +M https://doc.rust-lang.org/1.80.0/std/primitive.u8.html#impl-Copy-for-u8 https://doc.rust-lang.org/1.80.0/std/primitive.u8.html https://doc.rust-lang.org/1.80.0/std/primitive.u8.html 200 text/html 1121182 ee0cd4ff653be0fcb8144e42a9649a7320d7b617ef4301c0a86c187936072230 true 2026-08-01T01:28:40.161517Z +M https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html 200 text/html 85880 007591ab85643b155a8498f28b30a7fa11a392063c142abe8d7311b13b039d5b true 2026-08-01T01:28:40.212440Z +M https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html 200 text/html 11884 f783f80b1c0c29b052669635d24a767637a0b01dbc1c7c4f670154e59df3bbae true 2026-08-01T01:28:40.249944Z +M https://doc.rust-lang.org/1.80.0/std/primitive.u32.html#impl-Copy-for-u32 https://doc.rust-lang.org/1.80.0/std/primitive.u32.html https://doc.rust-lang.org/1.80.0/std/primitive.u32.html 200 text/html 1084587 ab99f0b7800c1ead37ea3857fbc8744f68d3ed1d0ddaf92fa49e395baf3bc146 true 2026-08-01T01:28:40.338022Z +M https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html 200 text/html 11904 ecb229ccdfc771ac09182b4abe0930e2b14013ad24c91340d7ad8e8350a3360a true 2026-08-01T01:28:40.377659Z +M https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#impl-Copy-for-u32 https://doc.rust-lang.org/1.82.0/std/primitive.u32.html https://doc.rust-lang.org/1.82.0/std/primitive.u32.html 200 text/html 1089970 15962c2e26e219d0df334635c8927ad7bb9007a38e237b529a61dce88aa4f4ef true 2026-08-01T01:28:40.469555Z +M https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html 200 text/html 86272 8f1114c00d2d8c08eea30caaa07257a7a018fd871a8d1e4598f55084be78a2e8 true 2026-08-01T01:28:40.520539Z +R https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +R https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#method.wrapping_add https://doc.rust-lang.org/1.82.0/std/primitive.u32.html https://doc.rust-lang.org/1.82.0/std/primitive.u32.html 200 text/html 1089970 15962c2e26e219d0df334635c8927ad7bb9007a38e237b529a61dce88aa4f4ef true 2026-08-01T01:28:40.469555Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.len https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.is_empty https://doc.rust-lang.org/1.82.0/std/primitive.slice.html https://doc.rust-lang.org/1.82.0/std/primitive.slice.html 200 text/html 828828 19f1230aa1d36c1e19eb9077a14bdb12b252d327ec3bab8c1f69d74a636a86ef true 2026-08-01T01:28:40.607857Z +K https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html#arithmetic-and-logical-binary-operators https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html 200 text/html 71217 9a08c53fb94c774ec5e4ddc069672f912f879716db7c17f649174afec3c72561 true 2026-08-01T01:28:40.659870Z +K https://doc.rust-lang.org/1.82.0/reference/types/numeric.html#integer-types https://doc.rust-lang.org/1.82.0/reference/types/numeric.html https://doc.rust-lang.org/1.82.0/reference/types/numeric.html 200 text/html 29770 7c21b867c4ba43f66984ece635049dedb96783d8de0490750cd2f1ee0fe44ead true 2026-08-01T01:28:40.703590Z +K https://doc.rust-lang.org/1.82.0/reference/items/traits.html#unsafe-traits https://doc.rust-lang.org/1.82.0/reference/items/traits.html https://doc.rust-lang.org/1.82.0/reference/items/traits.html 200 text/html 43949 8074f9255f3de0cc71edad5cbfbd22aeb8ac5427568c8478783d2aaca15d8363 true 2026-08-01T01:28:40.752604Z +K https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html#unsafe-traits-unsafe-trait https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html 200 text/html 33136 0de12b87f6a3967939624241cf67fc897de0eb93eca2b893c58dd1b734c4e0ea true 2026-08-01T01:28:40.797781Z diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/file-manifest.sha256 b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/file-manifest.sha256 new file mode 100644 index 0000000000..072b7e20e6 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/file-manifest.sha256 @@ -0,0 +1,47 @@ +a8ff68e702e9e4daa51a5560f39366f08e406df6eb8b1805484fa560a36cd0f8 fetch_authority.py +ad025876ef66f6e6578eea565088eaa99c24c0a603391d1e9f924858b3f63c48 freeze/allowlists/C.txt +b6a645f1b58c1ed0dab7480352ce18df11fe7a02468fd3df7fb6b403bc40d965 freeze/allowlists/K.txt +48b51034360f118b44292431a6776ee1eb24437abbd86a639e26fdd845cb3c70 freeze/allowlists/M.txt +3dc7862cdc2fdc1d44fd97235ad043602bd26ab3b428209b94fe9fb0602c14f8 freeze/allowlists/Q.txt +1cef79dd2eaef588d2343439d0b1a54b816adebf188a59fc6349f45055872ea4 freeze/allowlists/R.txt +c800f0c77ac630cbe60f1068cd172d32bc711d0a8ebe918c609996a8a3141977 freeze/allowlists/S.txt +c23fd5c190f565c18ee04ecf9918afd6463e434b573821b34022db16f594fa4e freeze/allowlists/W.txt +af068f9a92b8ae31e27cf9ee34075c54dd7f44906cc7e3ccb4b2369ad1e11b56 freeze/allowlists/X.txt +329634ba9421a9a65b2802b4ba5b07667fae194411904cbbfc7f002da84c0f97 freeze/authority-manifest.tsv +66522a17184a81a7832fc6a941bb947610c8971e29e28dbd7e64940542b198a2 freeze/manifest.json +982a13036918d561a6eb83d5effefe200ecdf81d6044b126c61c9017f1f91b2c freeze/oracle/controls.md +f6bddbbbd8edd032d257baf8c3b8d54731d1a3c0b284579dee9ec082887d5a30 freeze/oracle/domain.md +da2338aaed226d01c3a1a1399186dfc4b7c15b112321d65ddf883d405a56f961 freeze/oracle/verdict.md +76ed204e0bb03e546181bfef49f0d7cdf0c307c4d893e524e56820615cd8b0a7 freeze/plan.md +30df9df19f2dfa0056c0247f1d2e816bf0366d0f1d7eeacbea6755ed45f87117 freeze/policies/isolation.md +699fea44c59e993081c13fe9af7318e6a4b8bb4130b013b297f5256938b13405 freeze/policies/reruns.md +040a94fa045cc8baa4df28cddd1b7f18d4163561530351a73b1aa3578b86299d freeze/policies/tools.md +18e7b2bc357d82be6ca698d65a34bd73a53d321ce481e856a715b9b88bc47526 freeze/policies/word-count.md +1b67f22e8d221bdbc61d3ac700950b6fe140079a1c9d45736e1fe2cdb917abaf freeze/prompts/adjudicator.md +5c001fa4b647218d5c7a98468f66260b95d490fbdfc64b64217777b995df2782 freeze/prompts/report.md +ae7649ba7f57828ecff8f3c8c2b41987c65d67ff0cfb4142f7a17e08b7b55a5a freeze/prompts/scorer.md +39c4c657b39d2279dd3656e0f231906f3dfefdb853d24914616168b8c26c8bda freeze/randomization/commitments.json +75dbf874dbec04ee4738cfafa66d01ffba5510b81539662d8fb9596b43e3ae05 freeze/randomization/spec.md +3953c17ad001ee677061118952a5ac4fae44e21cc1ed1d64a0a0b2ae41c0049c freeze/rubrics/C.md +66ceb460ffb0893d9df8321707889c6f122fe0ccd9ec674fb31b07eb1ce5793e freeze/rubrics/K.md +d06d60ffd6b3478500caabb2ca88bb4b9327a7c63a3f0d913633ecb9a837d6e2 freeze/rubrics/M.md +154645b5d02c12a09d8162c6daca9d8843e81c9d4caa03165dc4acb5391d2a0a freeze/rubrics/Q.md +a3e9b8d28e8b9d4eafc5f9c4b553d6b862a623b2f786e1e5e806ea212317bc18 freeze/rubrics/R.md +d215ac1a4ed2e150f369b28aa12a4d77cf391c94ce47a98fccf132ad6a0c60f3 freeze/rubrics/S.md +b7ad63a8a816034852d1542e904eccce33c3e9ff60e6b4b2bb7f37f503ce8b82 freeze/rubrics/SCORER.md +c9aa4d310e2dc98d905caa3015c52c0cddbdcf95a1de65ea3a2f4fd78fef048c freeze/rubrics/W.md +ed88cb738258d69cf2e410f33fc2cabe7a209fecf36440b23707f3b9ebaeda36 freeze/rubrics/X.md +58d36e6a54002a29fe7f45c4f65d966a307212e6d082be75d34772fc5b128209 freeze/schemas/adjudication.schema.json +f606e610ef90dde0a7b5c425390ec44b6d4ce7059e95b2028feee9f0a8578381 freeze/schemas/disagreements.schema.json +d59caebb2439606adb0c64d1545732d3feaef93e5d10afa7eeba92d9a2623f3b freeze/schemas/event.schema.json +a54869ce1a3a4ac0cfa23ca63ec11a507406a5ae42ab09ad7a73cb5b444c729d freeze/schemas/final.schema.json +5b0563377e75d962d68ecace1225a73985fbb33ab422051377a0926cc7bb88d3 freeze/schemas/score.schema.json +1ebdd275e5b46a500de5028ffec6d30e48196b970cde80d07c72b73927038eba prepare.py +8d7ac565b05790805b4d79646c1ebb8dfd965f348ed05bf7c089276cc343aa0f protocol.py +8e739c0fcd05e19910d4dc986eebed31bd2629fc8d608ec6f86f15c38a6dfe75 sealed/blind-map.tsv +abd22fffee16a0629e9b721f2226c068a187949e05cba4711ee7a6bcbb5a02cb sealed/condition-map.tsv +03809f6698d6289e15f25434dbc931aed3619387d5a6ea586c0b0fc6d0f969ff sealed/launch-schedule.tsv +dfb7db6e6e6b35eb72034b9de3eece05e213e3e9ef2f285bca89d7352a5ea29b sealed/presentation-orders.tsv +7fec2919ec0b489c1c35a07b20123ffcde18f3a66958f87d87de4d7646a71ffd sealed/scoring-schedule.tsv +364231790dd4797101619bfd4bc1b538df0a194c7a066dddeba9fea2be7af368 sealed/seeds.json +b7809a7c771b203e0652b397ba9b57ef974424e474f87d5c740544715fdf7d84 sealed/target-map.tsv diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/manifest.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/manifest.json new file mode 100644 index 0000000000..6069c1f579 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/manifest.json @@ -0,0 +1,101 @@ +{ + "schema_version": 1, + "run_id": "2026-07-31-v3-targeted", + "status": "FROZEN_WHEN_LOCKED", + "prepared_utc": "2026-08-01T01:27:31Z", + "source_control": { + "repository_parent_commit": "47d01d20f04e4c5d4498a2f3ba3a098d370fd79c", + "branch": "unsafe-code-skill", + "note": "The freeze commit is created only after LOCK.json; every precommit freeze byte is independently bound by file-manifest.sha256." + }, + "objective": { + "primary": "Absolute confirmation of the frozen V3 capabilities represented by all eight mode rubrics.", + "diagnostic_comparator": "The coherent frozen V2 package; it is not a causal ablation.", + "release_claim": "A pass permits a later broad release gate but is not itself release readiness." + }, + "design": { + "modes": ["S", "C", "X", "Q", "W", "M", "R", "K"], + "conditions": 2, + "replicates_per_cell": 5, + "expected_reports": 80, + "balanced_waves": 5, + "maximum_concurrent_report_agents": 3, + "dual_blind_scorers_per_mode": 2, + "freshness": "fork_turns=none for every report, scorer, and adjudicator agent" + }, + "inference": { + "model": "gpt-5.6-sol", + "reasoning_effort": "ultra", + "service_tier": "priority", + "sampling_seed": null, + "hosted_model_build": null, + "limitation": "The orchestration API exposes neither a sampling seed nor an exact hosted model-build identifier." + }, + "packages": { + "v3": { + "role": "confirmatory candidate", + "tree_sha256": "668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf", + "skill_sha256": "0e23f7747cc63014bade7543efaf745e7e9a7e5d6dee2a48c602ef7a3eba091e" + }, + "v2": { + "role": "diagnostic comparator", + "tree_sha256": "40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897", + "skill_sha256": "a0a75ef8a14497aa78b50b459981097ee99605c57fec95c637cf59aaa20fe766" + } + }, + "targets": { + "S": "28ecc523e15b914a187814ab2752c0d85996948a552c62f970d8484bc6ed467a", + "C": "065c3cfc032af93e7576e17e49322826c4a379a707870175b16c2663d1e8e4e0", + "X": "25b4efef689601f3b5983bf6b914bd367153d0b21a6f1f4d40de50c5d412afa7", + "Q": "c0a4c43373a159cb38d08724af8b02187b249ab6a73f7e1b10b2276f38b0cb5a", + "W": "b27b95fbc9ffa9d335bb6b4614a9f227a5798122fca79b42cf53c9b106a6aff6", + "M": "b269cf068196d1c06b87be6bcded494827e7ac8a2cb6debf1eb0f0f5d0388479", + "R": "6d1a41909b012484d71f94d194071a7ebbb6773b7596db88127ca2a195b70ffc", + "K": "ca272e524b36a892e25f6169631a184ff764026451eff2f6ac4ab8e9d5e87ea2" + }, + "authority": { + "policy": "Only direct opens of exact URL-only per-mode allowlists are permitted; search and link traversal are prohibited.", + "manifest": "authority-manifest.tsv", + "allowlist_entries": 76, + "distinct_retrieved_pages": 46, + "retrieval_started_utc": "2026-08-01T01:28:38.381052Z", + "retrieval_finished_utc": "2026-08-01T01:28:40.797781Z", + "live_page_rule": "Immediately before each collection wave, fetch_authority.py must reproduce every frozen byte count, SHA-256, final URL, MIME type, and fragment. Earlier waves must be complete and no cell in the new wave may be prepared; drift blocks the run.", + "limitation": "Agents open live versioned pages rather than an evaluator-hosted mirror; network allowlisting is procedural on the shared host." + }, + "integrity": { + "tree_digest": "GNU tar 1.x: tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner -C TREE -cf - . | SHA-256; symlinks are rejected.", + "freeze_root": "SHA-256 of the byte-exact sorted file-manifest.sha256", + "randomization": "Five distinct nonzero 256-bit seeds and all derived maps are sealed and lock-covered; seed generation provenance is not externally attested.", + "events": "events.jsonl uses sequential canonical JSON records chained by SHA-256 of the preceding line; one run-wide operation lock serializes all protocol state transitions.", + "isolation": "Runtime paths and packets are neutral and inputs are read-only, but filesystem and network isolation are procedural rather than hardened." + }, + "canonical_inputs": { + "plan": "plan.md", + "oracles": ["oracle/domain.md", "oracle/verdict.md", "oracle/controls.md"], + "rubrics": "rubrics/", + "allowlists": "allowlists/", + "prompts": "prompts/", + "policies": "policies/", + "schemas": "schemas/", + "randomization": "randomization/", + "sealed_maps": "../sealed/", + "generator": "../prepare.py", + "authority_verifier": "../fetch_authority.py", + "protocol_and_aggregator": "../protocol.py" + }, + "primary_gates": { + "every_v3_atom": "PASS in 5 of 5 reports", + "v3_hard_errors": 0, + "v3_proposal_laundering": 0, + "v3_tcb_or_authority_defects": 0, + "v3_terminal_semantic_noncompletions": 0, + "v3_source_scope_or_budget_defects": 0 + }, + "known_nonclaims": [ + "No causal effect estimate follows from the coherent V3-versus-V2 comparison.", + "No cryptographic claim of condition blinding or packet-only access is made.", + "No target was built, tested, executed, or macro-expanded during fixture or freeze validation.", + "No report had been collected when this manifest was prepared." + ] +} diff --git a/evals/unsafe-rust/v3-targeted-drafts/controls.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/controls.md similarity index 85% rename from evals/unsafe-rust/v3-targeted-drafts/controls.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/controls.md index 2606f31bc9..f534317798 100644 --- a/evals/unsafe-rust/v3-targeted-drafts/controls.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/controls.md @@ -1,7 +1,6 @@ -# V3 Targeted Oracle Draft — R and K +# V3 Targeted Oracle — R and K -> Evaluator-only draft. Do not expose to report agents. Freeze only after two -> independent source and authority reviews. +> Evaluator-only material. Do not expose to report agents. ## R — abstraction redesign @@ -49,10 +48,13 @@ need. ### Hard errors -Hard error if the report trusts `Tail` as the only safe implementation, treats -safe trait prose as a caller obligation, calls the current API sound or merely -`UNPROVED` after closing the witness, certifies an unimplemented redesign, or -lets design intent narrow the current contract. +Hard error if the report: + +- trusts `Tail` as the only safe implementation; +- treats safe trait prose as a caller obligation; +- calls the current API sound or merely `UNPROVED` after closing the witness; +- certifies an unimplemented redesign; or +- lets design intent narrow the current contract. ### Authority @@ -121,16 +123,23 @@ each contract; do not infer the second disposition from the first. ### Hard errors -Hard error if the report trusts arbitrary safe `Position` implementations, -misses either safe surface while claiming complete coverage, calls either -surface sound, rejects the correct `last` implementation because its comment is -bad, explicitly calls the deficient comment adequate, relies on that comment as -the proof, claims complete requested coverage while omitting the reconstruction, -weakens the published trait contract from visible use alone, treats sealing -alone as satisfying the stated functional repair, claims a callable -caller-position API is repaired without locally coupling the selected position -to the particular slice, or certifies an unimplemented repair. A mere omitted -comment-quality disposition fails K4 but is not independently a hard error. +Hard error if the report: + +- trusts arbitrary safe `Position` implementations; +- misses either safe surface while claiming complete coverage; +- calls either safe surface sound; +- rejects the correct `last` implementation because its comment is bad; +- explicitly calls the deficient comment adequate; +- relies on that comment as the proof; +- claims complete requested coverage while omitting the reconstruction; +- weakens the published trait contract from visible use alone; +- treats sealing alone as satisfying the stated functional repair; +- claims a callable caller-position API is repaired without locally coupling + the selected position to the particular slice; or +- certifies an unimplemented repair. + +A mere omitted comment-quality disposition fails K4 but is not independently a +hard error. ### Authority diff --git a/evals/unsafe-rust/v3-targeted-drafts/domain.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/domain.md similarity index 99% rename from evals/unsafe-rust/v3-targeted-drafts/domain.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/domain.md index 2b93b8b8e7..174024dd31 100644 --- a/evals/unsafe-rust/v3-targeted-drafts/domain.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/domain.md @@ -1,4 +1,4 @@ -# V3 Targeted Domain Evaluator Draft +# V3 Targeted Domain Oracle > **Evaluator-only material.** Never expose this file, its mode labels, atoms, > intended predicates, hard-error rules, or authority list to an evaluated @@ -462,13 +462,13 @@ For effective rejection, verify that `cfg(all(...))` is true only when all listed predicates are true, that `#[cfg]` includes/removes its attributed form, and that `compile_error!` causes compilation to fail: -- `https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.84.0/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.84.0/std/macro.compile_error.html` -- `https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.85.0/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.85.0/std/macro.compile_error.html` -- `https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.86.0/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.86.0/std/macro.compile_error.html` @@ -509,7 +509,7 @@ Exact pages: - `https://doc.rust-lang.org/1.85.1/reference/patterns.html#or-patterns` - `https://doc.rust-lang.org/1.85.1/reference/patterns.html#wildcard-pattern` - `https://doc.rust-lang.org/1.85.1/std/macro.println.html` -- `https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#configuration-options` +- `https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#conditional-compilation` - `https://doc.rust-lang.org/1.85.1/reference/conditional-compilation.html#the-cfg-attribute` - `https://doc.rust-lang.org/1.85.1/std/macro.compile_error.html` @@ -525,4 +525,4 @@ review that entry, but do not enlarge it: - `https://doc.rust-lang.org/1.85.1/cargo/reference/features.html` No release blog, CI outcome, execution result, Miri result, prior report, or -this evaluator draft is an authoritative Rust semantic premise. +this evaluator oracle is an authoritative Rust semantic premise. diff --git a/evals/unsafe-rust/v3-targeted-drafts/verdict.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/verdict.md similarity index 96% rename from evals/unsafe-rust/v3-targeted-drafts/verdict.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/verdict.md index 462a52b776..1f3f340224 100644 --- a/evals/unsafe-rust/v3-targeted-drafts/verdict.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/oracle/verdict.md @@ -1,12 +1,10 @@ -# V3 Targeted Evaluator Draft — Verdict and Release Modes +# V3 Targeted Oracle — Verdict and Release Modes > **Evaluator-only material.** Never expose this file, its mode labels, atoms, > expected verdicts, or coverage relations to an evaluated report agent. > -> This is a draft until both authority reviewers verify every cited page and -> the final oracle and blind-scoring rubrics are frozen. Score propositions, -> not keywords or report layout. Equivalent explicit derivations pass. Do not -> infer a missing material link from vague shorthand. +> Score propositions, not keywords or report layout. Equivalent explicit +> derivations pass. Do not infer a missing material link from vague shorthand. ## Common scoring rules @@ -23,7 +21,7 @@ mode-specific rule below or the frozen global rules say so. ## Q — Existential refutation and missing unsafe dependency -Target: `fixtures/v3-targeted/q_quantifiers`. +Target: supplied Q snapshot. Let `C_Q` be Rust 1.80.0 times every target on which the exact source and pinned dependency resolve times every ordinary profile. @@ -85,7 +83,7 @@ Any of these is a hard error: - treating the unsafe dependency's documented caller precondition as though violating its prose were itself an executed UB event. -### Q authority to verify before freezing +### Q checked authority - `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` — exact proposition consumed: reaching @@ -96,7 +94,7 @@ No Rust authority is asserted for the absent dependency implementation. ## W — Whole-execution UB and independent behavioral refutation -Target: `fixtures/v3-targeted/w_whole_execution`. +Target: supplied W snapshot. Let `C_W` be Rust 1.80.0 times every target on which this exact source and the used Rust 1.80.0 standard-library item exist times every ordinary profile, and @@ -146,7 +144,7 @@ Any of these is a hard error: - falsely treating the independent input-one refutation as UB-containing, or claiming complete behavioral coverage while omitting it. -### W authorities to verify before freezing +### W checked authorities - `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` — reaching the function is undefined behavior. @@ -159,7 +157,7 @@ Any of these is a hard error: ## M — Multi-release affirmative certificates -Target: `fixtures/v3-targeted/m_multirelease`. +Target: supplied M snapshot. Let `C_M` be every supported target times every ordinary profile. For a function `f`, let `Calls_f` be all calls satisfying its documented safety @@ -288,7 +286,7 @@ Failure to establish a required positive regional result is an atom failure. It is not by itself a hard error unless the report also makes one of the false affirmative or scope-changing claims above. -### M authorities to verify before freezing +### M checked authorities Open each exact page and confirm the named description and Safety propositions, including all qualifications relevant to its exact release case: diff --git a/evals/unsafe-rust/v3-targeted-plan.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/plan.md similarity index 81% rename from evals/unsafe-rust/v3-targeted-plan.md rename to evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/plan.md index 19972dcab9..00d26c42c0 100644 --- a/evals/unsafe-rust/v3-targeted-plan.md +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/plan.md @@ -2,9 +2,10 @@ > **Evaluator-only material. Do not expose this file to evaluated agents.** > -> **Preregistration status:** DRAFT. No evaluated report may be collected until -> the packages, fixtures, prompts, oracle, rubrics, schedule, condition map, and -> gates are independently checked, frozen, and identified in the run manifest. +> **Preregistration status:** FROZEN. This status is effective only with a valid +> `LOCK.json` whose root matches `file-manifest.sha256`. No evaluated report may +> be collected before that lock exists and passes `protocol.py verify-static +> --locked`. ## Purpose @@ -45,7 +46,8 @@ Use eight focused modes, two conditions, and five fresh replicates per cell: Each evaluated agent receives one opaque package, one opaque target, and one empty output directory. Reports are randomized across conditions and modes. -Condition identity is revealed only after reports have been preserved, hashed, +The condition map is generated and sealed before collection. Condition identity +is revealed to the analysis only after reports have been preserved, hashed, blind-scored twice, and adjudicated. Five replicates are an engineering reliability minimum, not a population-level @@ -136,7 +138,8 @@ The targeted confirmation passes only if all of the following hold: 3. V3 has zero proposal laundering. 4. V3 silently admits no TCB premise and uses no invalid or inapplicable authority as a necessary proof premise. -5. Every V3 report respects the frozen source-only scope and word budget. +5. V3 has zero terminal semantic noncompletions. +6. Every V3 report respects the frozen source-only scope and word budget. Failure of any primary gate fails the run. Do not average failures away, weaken an atom after seeing reports, or use V2 weakness to excuse a V3 error. @@ -191,7 +194,19 @@ Before collection, freeze and hash: - one per-mode URL allowlist containing only exact official documentation URL identities, or a byte-identified mirror manifest; - the randomized schedule and condition map; -- output schema, word budgets, tool policy, and rerun policy. +- report-validation and aggregation programs, word budgets, tool policy, and + rerun policy. + +Every report, scorer, and adjudicator agent uses `gpt-5.6-sol` with reasoning +effort `ultra`, `fork_turns="none"`, and no helper agents. The orchestration API +does not expose a sampling seed or exact hosted model-build identifier; this is +an acknowledged reproducibility limit. Collection follows the sealed schedule +with at most three report agents active and a complete balanced-wave barrier. +Immediately before preparing each wave, the authority verifier must reproduce +the frozen official-document bytes. That wave may not be verified early: every +earlier wave must already be complete and no cell in the new wave may yet be +prepared. Protocol state transitions are serialized by one run-wide operation +lock. Each report agent must: @@ -201,20 +216,34 @@ Each report agent must: permitted official Rust/std pages (or frozen mirror bytes), and empty output directory; - avoid building, testing, executing, or macro-expanding the target; -- write one `report.md` and return the same report; +- write one canonical `report.md`; the final chat response is preserved only as + operational metadata and is never scored; - stay within 1,800 words, except that X receives a 2,400-word cap and K a 2,200-word cap, identical across conditions. Only genuine infrastructure failures may be rerun. Budget exhaustion, refusal, or semantic noncompletion is an incomplete/failed replicate, not infrastructure. Preserve every invalid attempt and document the disposition before retrying. +A terminal report noncompletion is blind-scored as produced; when no usable +`report.md` exists it receives an evaluator-marked canonical placeholder so +missing propositions fail without inventing agent work, and it independently +fails the zero-semantic-noncompletion gate. A non-rerunnable +invalid scorer or adjudicator output makes the run `INVALID`; no replacement +judgment is fabricated. + +The same attempt lifecycle applies to scorers and adjudicators. An API failure +before any agent identity exists is recorded but is not an attempt. A genuine +infrastructure failure after start preserves that numbered attempt and permits +exactly the next fresh attempt. Any non-infrastructure invalid evaluator output +is terminal and makes the run `INVALID`. ## Blind scoring and adjudication After collection and before unblinding: 1. Preserve and hash every raw report. -2. Assign random anonymous labels independently within each mode. +2. Materialize the pre-frozen anonymous-label map independently within each + mode. 3. Give two fresh scorers the target, common scoring rules, exact per-mode rubric, and ten anonymous reports, but no package, condition map, sibling package, or prior scores. @@ -223,6 +252,11 @@ After collection and before unblinding: 6. Adjudicate novel findings against source and authority before unblinding. 7. Preserve raw scores, adjudications, ledgers, and all integrity checks. +Blind scorers decide only scope defects visible in report content. The runner +independently counts words and records every known operational path/tool/source +deviation. Aggregation ORs those three sources into the source-scope/budget +gate; no scorer is asked to infer unavailable execution telemetry. + The scorer must not infer a missing material premise from vague shorthand. The rubric must state in advance which compact formulations count, especially where one conceptual defect admits multiple independently scored witnesses. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/isolation.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/isolation.md new file mode 100644 index 0000000000..ec4a61c77a --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/isolation.md @@ -0,0 +1,44 @@ +# Frozen Isolation and Packet Policy + +Each report cell receives a unique neutral runtime root with entries named only +`package`, `target`, `allowlist.txt`, and `output`. Inputs are copied +byte-for-byte, made read-only where the host permits, and verified against +frozen identities before launch and after the attempt. `output` begins empty. + +The shared collaboration host does not provide a hardened per-agent mount or +network allowlist. Isolation therefore remains procedural. Runtime inputs must +not contain a repository, `.git`, evaluator files, mode/condition names, or +cross-cell paths. This limitation is disclosed in every result and prevents a +claim of cryptographically enforced blinding. + +Before scorer launch, preserve and hash each canonical `report.md` and copy it +into a fresh mode packet under its frozen anonymous label. Chat-return text is +operational metadata and is not an alternative report channel. Prefer neutral +paths from collection so no report redaction is needed. Any required +normalization must use a frozen transformer and preserve the exact original, +transformed hash, and diff. + +Scorer packets contain only the target, allowlist, common rules, one mode +rubric, ten anonymous reports, score schema, and packet-local hashes. All packet +filesystem timestamps are normalized to the Unix epoch. Packets contain no +package, condition, run ID, collection order, agent identity, prior score, or +sibling mode. Adjudicator packets contain only the disputed reports/cells and +rationales, not unrelated agreed decisions. + +The append-only event ledger externally pins the canonical collection index +and every complete scorer and adjudicator packet byte tree. Packet verification +checks that external digest as well as the packet-local manifest before every +use. One run-wide operation lock serializes all state checks, artifact writes, +and event transitions. A canonical report, score, or adjudication and its +attempt record derive from the same single capture of the agent's output bytes; +the protocol does not reread a live output path to create either copy. + +Failure preservation remains possible when an evaluated agent changes a +runtime input despite the procedural restrictions. The protocol binds the +expected neutral path, records expected and safely observed input identities +without following symlinks, and snapshots the entire neutral runtime on an +input or inventory verification failure. The snapshot preserves every regular +byte and records every directory, symlink, and special entry without following +symlinks. It also snapshots the central setup or source packet whose identity +was checked. Report input drift becomes a terminal scope failure; evaluator +packet drift makes the run `INVALID`. It never authorizes a retry. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/reruns.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/reruns.md new file mode 100644 index 0000000000..8376a064f5 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/reruns.md @@ -0,0 +1,43 @@ +# Frozen Attempts, Reminders, and Rerun Policy + +Every attempt is immutable and receives its own directory. Preserve every +regular filesystem output byte, agent identity, timestamps, hashes, known +tool/path deviations, API completion/error state, and disposition before +considering another attempt. The canonical file is the sole evaluated channel; +chat-return prose need not be duplicated byte-for-byte. + +Only an externally established infrastructure failure may authorize a fresh +agent attempt. Examples are a service error before semantic work, a tool crash, +or a write failure caused by unavailable infrastructure. Refusal, word-budget +exhaustion, timeout after substantive work, invalid reasoning, missing atoms, +or other semantic noncompletion is a failed replicate and may not be rerun. + +An API failure that produces no agent identity is not an evaluated attempt; +record it separately as `API_NO_AGENT_START`, then launch the same next attempt +number. This applies to report, scorer, and adjudicator launches. For any +started attempt, use only these infrastructure disposition codes: +`SERVICE_ERROR_BEFORE_OUTPUT`, `ORCHESTRATOR_TOOL_FAILURE`, and +`FILESYSTEM_FAILURE`. Record the observed evidence and preserve every partial +artifact. A condition not fitting one of those codes is not rerunnable. + +A started scorer or adjudicator infrastructure failure likewise preserves its +own attempt directory and authorizes exactly the next numbered fresh attempt. +A schema-invalid, semantically incomplete, or otherwise non-infrastructure +scorer/adjudicator output is non-rerunnable and makes the evaluation `INVALID`. + +A terminal report-agent noncompletion is represented by its usable `report.md` +when one exists or by an evaluator-marked placeholder otherwise, is blind-scored +under the ordinary rubric, and is never rerun. Never fabricate a replacement +report, score, or adjudication. An empty or whitespace-only `report.md` is +semantic noncompletion, never a complete report. + +Preserved evaluator-attempt and invalid-output directories are inventoried in +both directions against unique ledger events. Any orphan directory, missing +directory, duplicate event, invalid hierarchy, `INVALID.json` marker, or +terminal-invalid ledger event fails closed. Once either terminal-invalid signal +exists, no later evaluation operation is permitted; a complete invalid state +must bind the marker, attestation, return event, and invalidation event exactly. + +One exact neutral reminder from `prompts/report.md` is permitted at 180 seconds. +No second reminder or substantive steering is permitted. Scorers and +adjudicators receive no reminders. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/tools.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/tools.md new file mode 100644 index 0000000000..5c1c8749ad --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/tools.md @@ -0,0 +1,16 @@ +# Frozen Tool and Source Policy + +Report agents may read only their per-cell package, target, URL-only allowlist, +allowed exact pages, and initially empty output directory. They may write only +`output/report.md` with `apply_patch`. Direct URL opens are permitted; web +search, link traversal, and every non-allowlisted page are prohibited. + +Targets must not be modified, built, tested, executed, or macro-expanded by +execution. Report agents may not spawn helper agents. They must not inspect +sibling/enclosing paths, repository or Git data, evaluator material, schedules, +maps, other cells, or prior reports. + +Scorers and adjudicators have analogous packet-only read scope and one-file +output scope. Tool and path restrictions are procedural on the shared host; +they are not a hardened sandbox. Every known deviation is preserved in the +append-only event ledger and fails the scope gate when material. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/word-count.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/word-count.md new file mode 100644 index 0000000000..c6b9eadd8c --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/policies/word-count.md @@ -0,0 +1,22 @@ +# Frozen Word-Count Policy + +Decode `report.md` as UTF-8, split it on every Unicode whitespace run using +Python 3 `str.split()`, and count the resulting nonempty fields. Invalid UTF-8 +is an invalid attempt. + +The caps are: + +| Mode | Maximum words | +|---|---:| +| S | 1,800 | +| C | 1,800 | +| X | 2,400 | +| Q | 1,800 | +| W | 1,800 | +| M | 1,800 | +| R | 1,800 | +| K | 2,200 | + +The same cap applies to both conditions. A report above its cap remains +preserved but fails the source-scope/budget gate; it is not an infrastructure +failure and may not be rerun. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/adjudicator.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/adjudicator.md new file mode 100644 index 0000000000..52f90928e1 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/adjudicator.md @@ -0,0 +1,23 @@ +# Frozen Blind-Adjudicator Prompt + +```text +Act as a fresh blind adjudicator. Read every file under [PACKET], including the +target, URL allowlist, SCORER.md, RUBRIC.md, adjudication schema, disputed +anonymous reports, DISAGREEMENTS.json, and the two rationales for each disputed +cell. Follow the frozen instructions exactly. + +Resolve only the listed disagreements. Preserve every agreed decision. Decide +from the report's actual evidence: external authority may verify a premise the +report invokes, but may not add a material premise or derivation the report +omitted. Do not decide by majority, style, report similarity, or speculation +about a generating condition. + +Inspect only [PACKET] and exact URLs in its allowlist. Do not inspect siblings, +the enclosing repository, packages, condition maps, other scores, or other +adjudications. Do not modify the packet and do not spawn helper agents. + +Write exactly one UTF-8 JSON file, [OUTPUT]/adjudication.json, conforming to the +supplied schema. Create no other output file. That file is the sole evaluated +artifact; keep any final chat response to a terse operational confirmation. +Give compact decisive evidence and record genuine rubric ambiguity. +``` diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/report.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/report.md new file mode 100644 index 0000000000..2669f5bbba --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/report.md @@ -0,0 +1,36 @@ +# Frozen Evaluated-Agent Prompt + +Each report agent receives this text byte-for-byte except for resolving the +five bracketed placeholders. `[WORD_LIMIT]` is mode-specific but identical +between conditions within a mode. + +```text +Act as a fresh source-review agent. Read the complete unsafe Rust skill package +rooted at [PACKAGE]/SKILL.md and every reference it directs you to for this +task, then follow it. Read REQUEST.md and every file in [TARGET], and complete +exactly the requested review or design work. + +Your only permitted local inputs are [PACKAGE], [TARGET], [URL_ALLOWLIST], and +the empty [OUTPUT] directory. The allowlist contains the only external pages +you may inspect. Open those exact URLs directly if needed; do not search the +web or follow links to another page. Do not inspect a sibling or enclosing +directory, repository or version-control material, evaluator files, another +package or target, prior reports, schedules, or condition information. Treat +paths and opaque labels as non-semantic. + +Do not modify, build, test, execute, or macro-expand the target. Do not spawn +helper agents. Write exactly one UTF-8 file, [OUTPUT]/report.md, using +apply_patch, and create no other output file. That file is the sole evaluated +artifact; keep any final chat response to a terse operational confirmation. +Keep the report at or below [WORD_LIMIT] words, counting the nonempty fields +produced by splitting Unicode text on whitespace. This is a focused source +review; provide the complete proof material compactly. +``` + +No substantive steering is permitted. If the agent is still running 180 +seconds after launch, exactly one reminder may be sent: + +```text +Complete now within the frozen word limit using only material already +inspected; do not widen scope. +``` diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/scorer.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/scorer.md new file mode 100644 index 0000000000..bc42474f76 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/prompts/scorer.md @@ -0,0 +1,28 @@ +# Frozen Blind-Scorer Prompt + +Each scorer receives this text byte-for-byte except for resolved `[PACKET]` and +`[OUTPUT]` paths and its opaque `[SCORER_ID]`. + +```text +Act as a fresh blind evaluator. Read every file under [PACKET], including the +target, URL allowlist, SCORER.md, RUBRIC.md, score schema, and anonymous reports +A through J. Follow the frozen scoring instructions exactly and score every +report independently. Inspect reports in the `presentation_order` recorded in +PACKET.json; do not reorder them by similarity or suspected condition. + +Inspect only [PACKET] and the exact URLs in its allowlist. Open an allowed URL +directly if needed; do not search, follow links, or inspect any sibling or +enclosing directory, repository material, package, condition map, prior score, +or another scorer's output. Do not identify, cluster, or speculate about report +conditions. Do not modify the packet and do not spawn helper agents. + +Write exactly one UTF-8 JSON file, [OUTPUT]/score.json, conforming to the +supplied schema and using scorer_id [SCORER_ID]. Create no other output file. +That file is the sole evaluated artifact; keep any final chat response to a +terse operational confirmation. Score every atom and every independent defect +flag, cite compact report evidence for each decision, and record genuine rubric +ambiguity. Keep all evidence concise. +``` + +No reminder is permitted. An invalid or incomplete scorer output is preserved +and does not silently replace an independent score. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/commitments.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/commitments.json new file mode 100644 index 0000000000..8c0478a3e0 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/commitments.json @@ -0,0 +1,11 @@ +{ + "algorithm": "sha256(tag_utf8 || NUL || seed_bytes)", + "commitments": { + "blind": "f5c2a105dcd39c9a18b18e24a8b2393e9393a2d8cc9e2edd5bae3fa80fd32ed5", + "condition": "eb32738dca411d321e0bdd76b54a7ff82eedd0044d1bf1ce00de8bec201d190d", + "presentation": "6ee4618cc2757cd4a1fbfbcac0477247c3b346e73574610c62345ae07b9b1aa2", + "schedule": "23b61a3ae86f32f6262c96b46e6b5f8002d2a018c742ab7c371a58fb0c6ffe16", + "scorer": "395f3040683208aaf77eef98536646efdb83d9504bb0db00c3520b202ca587de" + }, + "schema_version": 1 +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/spec.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/spec.md new file mode 100644 index 0000000000..506a860055 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/randomization/spec.md @@ -0,0 +1,41 @@ +# Frozen Randomization Specification + +`prepare.py` is the executable specification. It uses five independently +generated 256-bit seeds, stored under evaluator-only `sealed/seeds.json`, and +domain-separated SHA-256 hash sorting. No language PRNG or iteration-order +behavior determines an order. + +For tag `T`, seed `S`, and canonical UTF-8 value `V`, the sort key is: + +```text +SHA256(T || NUL || bytes_from_hex(S) || NUL || V) +``` + +The canonical tuple breaks the cryptographically negligible event of a key +collision. + +The procedure is: + +1. Hash-sort real condition roles and assign opaque labels `c0` and `c1`. +2. Hash-sort mode names and assign opaque target labels `m0` through `m7`. +3. Treat each replicate as a balanced wave containing all eight modes and both + conditions. Hash-sort the five waves, then the sixteen cells within each + wave. Assign sequential operational run IDs only after sorting. +4. Derive each neutral 128-bit runtime cell ID from a separately tagged hash of + its canonical `(mode, role, replicate)` tuple. Assert all 80 are unique. +5. Independently hash-sort each mode's ten run IDs and assign blind labels A–J. +6. Independently hash-sort A–J for each scorer's presentation order. +7. Independently hash-sort the sixteen scorer claims for the scoring launch + order. + +The sealed maps are frozen before collection but never enter a report-agent or +blind-scorer packet. Because the shared repository and host are not hardened +against deliberate out-of-scope reads, secrecy is procedural. Commitments are +recorded in `commitments.json`; the seed and generated-map bytes are covered by +the freeze lock. + +Collection processes one complete balanced wave before the next. Within a +wave, starts follow `launch-schedule.tsv`; up to three report agents may be +active concurrently. A later service limitation may reduce concurrency but may +not reorder starts or cross the wave barrier. Every actual start/completion is +recorded in the append-only event ledger. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/C.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/C.md new file mode 100644 index 0000000000..cb18a8c259 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/C.md @@ -0,0 +1,128 @@ +# Mode C Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## C — Nonlinear Conflicting Policies + +### Intended predicates + +Use the fixture's `V`, `X`, `A`, `W`, `f`, and `h` notation. The two published +predicates are exactly: + +```text +P_Scarlet(v,t,f,h) + iff v in V and t in {X,A,W} and + (!f + or (f and t = X and (!h or v >= 1.85.0)) + or (f and t = A and h)). + +P_Indigo(v,t,f,h) + iff v in V and t in {X,A,W} and + (!f + or (f and t = X and (h or v >= 1.86.0)) + or (f and t = A and !h and v >= 1.85.0)). +``` + +Neither predicate contains the other. Representative disputed cases include: + +- Scarlet-only: `(1.84.0, X, turbo, !hardened)` and + `(1.84.0, A, turbo, hardened)`; +- Indigo-only: `(1.84.0, X, turbo, hardened)` and + `(1.85.0, A, turbo, !hardened)`. + +There is no authorized exact project predicate. The minimal conservative audit +domain is the positive-claim union, not a newly selected project promise: + +```text +U_C = P_Scarlet union P_Indigo + +U_C(v,t,f,h) + iff v in V and t in {X,A,W} and + (!f + or (f and t = X) + or (f and t = A and (h or v >= 1.85.0))). +``` + +Thus every non-`turbo` configuration in the base target set is in `U_C`; +every `turbo` x86 configuration is in it; a `turbo` aarch64 configuration is +in it exactly when `h` is true or `v >= 1.85.0`; and no `turbo` wasm32 +configuration is in it. + +For the conservative source theorem, include every valid `Option` input, +every Cargo profile, and both debug-assertion states: + +```text +Required_C = U_C cross O_S cross P cross D. +``` + +With applicable versioned `Option` propositions, the source proof is +parametric over this larger accepted-source region: + +```text +Covered_C = { !f on X/A/W, or f on X/A } cross O_S cross P cross D. +``` + +`h` and the exact version do not affect the source derivation. Therefore +`Required_C subset-of Covered_C` without enumerating `V x T x f x h x P x D`. + +### Expected verdicts + +- Identity of the controlling exact project support predicate: **UNRESOLVED** + (or equivalently `UNPROVED` as a policy-identification claim). +- Safe-API soundness and documented behavior over conservative union `U_C`: + **PROVED relative to `BUILD-MAP-C`**. +- The conclusion may state that either possible project promise is sound, + because both are subsets of the proved conservative domain. It must not + relabel that audit domain as the recovered exact project promise. +- `turbo` plus wasm32: outside both policies and effectively rejected by the + active `compile_error!` configuration, **PROVED relative to `BUILD-MAP-C`**. + +The non-`turbo` path uses the documented safe `unwrap_or`. The `turbo` path +returns on `None`; reaching `unwrap_unchecked` therefore establishes `Some`, +and the returned byte satisfies the same public behavior. This proof is +independent of target, `hardened`, profile, and debug assertions. + +### Atoms + +- **C1 — Both nonlinear predicates:** Reproduces both policies without + choosing precedence and identifies at least one Scarlet-only and one + Indigo-only region. +- **C2 — Conservative domain:** Takes either rigorous route: derive the exact + minimal `U_C` formula, including its conditional aarch64 clause; or choose a + transparent audit superset `S` and prove separately that + `P_Scarlet subset-of S` and `P_Indigo subset-of S`. In either route, do not + call the chosen conservative domain the unresolved exact project promise. +- **C3 — Enforced exclusion:** Recognizes that `turbo` wasm32 is excluded by + both policies and that the `compile_error!` selected by the matching `cfg` + makes the ordinary supported build interface reject it, using only the exact + accepted `BUILD-MAP-C` mappings and applicable Rust semantics. +- **C4 — Soundness certificate:** Proves the safe non-`turbo` branch and the + `None`-return/reachable-`Some` argument for the unsafe `turbo` branch, + expresses the resulting `Covered` region parametrically, proves it contains + the chosen conservative audit domain, and reports soundness **PROVED relative + to `BUILD-MAP-C`** over that domain. +- **C5 — Behavioral certificate:** Proves both branches return zero for `None` + and the contained byte for `Some`, shows the same `Covered` region contains + the chosen conservative audit domain, and reports the documented behavior + **PROVED relative to `BUILD-MAP-C`** over that domain. +- **C6 — Policy status:** Reports the controlling exact project predicate + **UNRESOLVED** and does not relabel the chosen conservative audit domain as + that promise. + +### Hard-error triggers + +A report commits a hard error if it: + +- silently selects Scarlet or Indigo, intersects them, or asserts that their + union is the recovered exact project promise; +- claims that a chosen conservative audit domain contains both published + predicates, or claims closure over it, when the stated derivation does not; + merely declining closure is an atom failure, not a hard error; +- treats `turbo` wasm32 as supported, or claims it is effectively rejected + without accounting for the selected `compile_error!`; +- treats Cargo's minimum, a hypothetical build sample, or a policy sentence as + a Rust semantic axiom; +- invents or widens a build-tool mapping beyond accepted `BUILD-MAP-C`; +- claims `PROVED` while leaving any part of the chosen conservative domain + uncovered; or +- calls the shown source `UNSOUND` or `CONTRACT-BROKEN`. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/K.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/K.md new file mode 100644 index 0000000000..4d3bcb7133 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/K.md @@ -0,0 +1,97 @@ +# Mode K Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## K — multi-issue regression + +### Exact results + +Within the exact target/profile scope declared in `REQUEST.md`, safe +implementations of `callback_index::Position` may return any `usize`. For a +one-byte slice, an implementation returning 1 gives valid safe +calls to both `read` and `write`, each of which executes its respective +out-of-bounds unchecked operation. Both safe surfaces are independently +`UNSOUND`. + +`local_proof::last` is sound. On the nonempty branch, `len > 0`, so +`index = len - 1` is defined and `index < len`; that discharges +`get_unchecked(index)`. The existing comment states none of this and does not +identify the callee obligation. The report must expose the reconstructed proof +and separately classify the implementation as proved and the proof artifact as +deficient. + +For `published_lane`, the unsafe trait contract makes valid implementations +responsible for both `INDEX < 2` and the exact `NAME`/`INDEX` relation. The shown +`High` implementation satisfies both clauses. `read` consumes only the first +clause and is sound for every valid implementation. Unknown downstream +consumers mean the unused `NAME` clause cannot be weakened from the published +1.x contract merely because `read` does not consume it. + +### Atoms + +- **K1:** For `callback_index::read`, construct the arbitrary-safe-impl witness, + prove the length-1/index-1 precondition failure and UB consequence, and report + this safe surface `UNSOUND`. +- **K2:** Independently give the same complete disposition for + `callback_index::write`; do not stop after K1. +- **K3:** For `local_proof::last`, reconstruct the nonempty -> `len > 0` -> + `len - 1 < len` derivation, connect it to the exact unchecked-index contract, + and report the implementation obligation proved. +- **K4:** Separately report the existing `local_proof::last` safety comment + materially deficient and provide replacement proof content that identifies + the unchecked-index obligation and the K3 derivation. +- **K5:** Prove that `High` satisfies both clauses of its unsafe-trait contract: + `INDEX < 2` and the exact `NAME`/`INDEX` relation. +- **K6:** Prove `published_lane::read` sound for every valid implementation of + the complete `Lane` contract by consuming its `INDEX < 2` clause. +- **K7:** Retain the `NAME` clause and explain that unknown public + implementers/consumers prevent weakening the published 1.x contract merely + because the only visible consumer uses `INDEX`. +- **K8:** Propose a repair that preserves callable read and write operations at + positions chosen by safe callers and locally couples and checks each chosen + position against the particular slice. Deleting or making the APIs + uncallable, or merely sealing or eliminating the caller-implementable trait, + does not meet that functional requirement. + +Score each atom independently. Compact shared reasoning may serve K1 and K2 +only if it explicitly names both operations and shows that the witness reaches +each contract; do not infer the second disposition from the first. + +### Hard errors + +Hard error if the report: + +- trusts arbitrary safe `Position` implementations; +- misses either safe surface while claiming complete coverage; +- calls either safe surface sound; +- rejects the correct `last` implementation because its comment is bad; +- explicitly calls the deficient comment adequate; +- relies on that comment as the proof; +- claims complete requested coverage while omitting the reconstruction; +- weakens the published trait contract from visible use alone; +- treats sealing alone as satisfying the stated functional repair; +- claims a callable caller-position API is repaired without locally coupling + the selected position to the particular slice; or +- certifies an unimplemented repair. + +A mere omitted comment-quality disposition fails K4 but is not independently a +hard error. + +### Authority + +- Rust 1.82.0 slice `get_unchecked`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked` +- Rust 1.82.0 slice `get_unchecked_mut`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut` +- Rust 1.82.0 slice `len` and `is_empty`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.len` + and + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.is_empty` +- Rust 1.82.0 Reference, arithmetic binary operators and integer overflow: + `https://doc.rust-lang.org/1.82.0/reference/expressions/operator-expr.html#arithmetic-and-logical-binary-operators` +- Rust 1.82.0 Reference, unsigned integer value domains: + `https://doc.rust-lang.org/1.82.0/reference/types/numeric.html#integer-types` +- Rust 1.82.0 Reference, unsafe traits: + `https://doc.rust-lang.org/1.82.0/reference/items/traits.html#unsafe-traits` +- Rust 1.82.0 Reference, unsafe-trait proof obligations: + `https://doc.rust-lang.org/1.82.0/reference/unsafe-keyword.html#unsafe-traits-unsafe-trait` diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/M.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/M.md new file mode 100644 index 0000000000..dbd98585cc --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/M.md @@ -0,0 +1,171 @@ +# Mode M Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## M — Multi-release affirmative certificates + +Target: supplied M snapshot. + +Let `C_M` be every supported target times every ordinary profile. For a +function `f`, let `Calls_f` be all calls satisfying its documented safety +preconditions. The exact required domains are: + +- `Required(M-ack) = V_ack times C_M times Calls_ack`, where + `V_ack = {1.80.0, 1.80.1, 1.81.0, 1.82.0}` and every typed call is valid; +- `Required(M-store) = V_store times C_M times Calls_store`, where + `V_store = {1.80.0, 1.81.0}`; +- `Required(M-copy) = V_copy times C_M times Calls_copy`, where + `V_copy = {1.80.0, 1.80.1, 1.81.0, 1.82.0}`; and +- `Required(M-load) = V_load times C_M times Calls_load`, where + `V_load = {1.80.0, 1.80.1, 1.81.0, 1.82.0}`. + +For `store_word`, `copy_byte`, and `load_word`, the same release domain applies +separately to soundness and the documented normal-return postcondition. + +### Expected results and atoms + +- **M1 — `acknowledge` domain:** Preserve exact finite set `V_ack` and its + target/profile product; every well-typed call is in `Calls_ack` because the + API adds no safety precondition. +- **M2 — `store_word` domains:** Preserve exact finite set `V_store` and its + target/profile/valid-call product separately for soundness and the documented + normal-return postcondition. +- **M3 — `copy_byte` domains:** Preserve exact finite set `V_copy` and its + target/profile/valid-call product separately for soundness and the documented + normal-return postcondition. +- **M4 — `load_word` domains:** Preserve exact finite set `V_load` and its + target/profile/valid-call product separately for soundness and the documented + normal-return postcondition. + +For M1–M4, do not let the cutoff add or remove releases and do not substitute +an unscoped crate-wide claim for the exact products. + +- **M5 — Parametric `acknowledge` proof:** Verify that accepted general entry + `SEM-EMPTY-BLOCK-180-182` has exactly the required release, target, and + profile scope. Independently inspect the local syntax and establish that the + exact body is an empty block. For arbitrary + `(v, target, profile, call) in Required(M-ack)`, combine only those premises: + `acknowledge` is zero-parameter and unit-returning, the valid call has defined + callee evaluation and returns `()`, and the `unsafe fn` marker changes only + its static caller obligation. Thus + `Covered(M-ack) = Required(M-ack)` and source-level soundness is **PROVED + parametrically relative to `SEM-EMPTY-BLOCK-180-182`**. Keep the admission + conspicuous; the TCB entry is general semantics, not a target-specific + assertion that the function is sound. +- **M6 — `store_word` soundness partition:** The 1.80.0 `ptr::write` authority + applies exactly to the 1.80.0 case and the 1.81.0 authority to the 1.81.0 + case. For each case, the documented caller contract entails the applicable + page's alignment and write-validity preconditions. The identity + `V_store = {1.80.0} union {1.81.0}` proves exhaustiveness. Therefore + `Covered(M-store-sound) = Required(M-store-sound)` and soundness is + **PROVED** by an exact finite partition. +- **M7 — `store_word` postcondition partition:** In each exact release case, + the applicable page says that `ptr::write(dst, value)` writes the supplied + `value` to `dst` without reading or dropping the old value. The same finite + partition covers every normal-return postcondition obligation, so + `Covered(M-store-post) = Required(M-store-post)` and the documented + postcondition is **PROVED**. +- **M8 — `copy_byte` soundness under the exact TCB:** Verify the Rust 1.80.0 + `copy_nonoverlapping` safety proposition, primitive `u8` size, `u8: Copy`, + and exact `Copy` semantics base propositions. Then apply only accepted entry + `COMPAT-COPY-180-182`, with its exact release set, `T = u8`, `count = 1`, + target/profile domain, and consumer. The caller contract entails its source + and destination validity, initialization, alignment, and nonoverlap clauses; + the admitted `u8` and `Copy` facts establish the one-byte specialization and + avoid the ownership hazard for non-`Copy` values. Thus + `Covered(M-copy-sound) = Required(M-copy-sound)` and soundness is **PROVED + relative to `COMPAT-COPY-180-182`**. +- **M9 — `copy_byte` postcondition under the exact TCB:** The accepted entry + preserves the base proposition that the call copies the source byte into the + destination while leaving the source byte unchanged; `u8` has size one and + implements `Copy` with the cited copy rather than move semantics. It covers + every normal-return postcondition obligation, so + `Covered(M-copy-post) = Required(M-copy-post)` and the documented + postcondition is **PROVED relative to `COMPAT-COPY-180-182`**. +- **M10 — `load_word` soundness remainder:** In the 1.80.0 and 1.82.0 endpoint + cases, the exact `ptr::read` Safety sections plus exact-version `u32: Copy` + and `Copy`-semantics facts prove soundness from the caller contract, + including safe retention of the source value alongside the returned copy. + Thus + `Covered(M-load-sound) = {1.80.0, 1.82.0} times C_M times Calls_load`. + The 1.80.1 and 1.81.0 regions and therefore the complete `V_load` soundness + claim are **UNPROVED**. No UB witness is supplied, so `UNSOUND` does not + follow. +- **M11 — `load_word` postcondition remainder:** In the same two endpoint + cases, the applicable `ptr::read` descriptions establish that the returned + value is read from `src` while the source is left unchanged; the exact + `u32: Copy` and `Copy`-semantics facts discharge the ownership qualification. + Therefore + `Covered(M-load-post) = {1.80.0, 1.82.0} times C_M times Calls_load`. + The two interior regions and the complete postcondition claim are + **UNPROVED**. No defined wrong-result witness is supplied, so + `CONTRACT-BROKEN` does not follow. + +Scoring guidance, not a separate atom: if a report actually relies on endpoint +sampling, the cutoff, an unstated stability guarantee, backward projection, +evidence for another operation, or a widened TCB entry to fill the `ptr::read` +interior, reject that basis explicitly. For an `UNPROVED` interior, require the +smallest missing premise: a verified authority for the interior releases, an +exact applicable accepted compatibility entry, or another complete parametric +proof. A report that uses none of the listed invalid bases need not recite them. + +### M hard errors + +Any of these is a hard error: + +- reporting the complete `load_word` claim `PROVED` by endpoint sampling, + cutoff reasoning, a stability badge, unprovided interior docs, or another + operation's evidence; +- claiming `Required subset Covered` for `load_word` while omitting either + interior release; +- silently widening `COMPAT-COPY-180-182` by release, operation, type/count, + target/profile domain, proposition, or consumer; +- silently widening `SEM-EMPTY-BLOCK-180-182` by release, source shape, + proposition, or consumer; +- treating either accepted entry as authoritative Rust text rather than a + conspicuous TCB premise; +- projecting a later Rust proposition backward to an earlier release without + an exact applicable premise; or +- issuing `UNSOUND` or `CONTRACT-BROKEN` for an unresolved interior merely from + failure to prove it. + +Failure to establish a required positive regional result is an atom failure. It +is not by itself a hard error unless the report also makes one of the false +affirmative or scope-changing claims above. + +### M checked authorities + +Open each exact page and confirm the named description and Safety propositions, +including all qualifications relevant to its exact release case: + +- `https://doc.rust-lang.org/1.80.0/std/ptr/fn.write.html` + — overwrites without reading or dropping the old value; `dst` must be valid + for writes and properly aligned. +- `https://doc.rust-lang.org/1.81.0/std/ptr/fn.write.html` + — the same exact propositions for the separate 1.81.0 case. +- `https://doc.rust-lang.org/1.80.0/std/ptr/fn.copy_nonoverlapping.html` + — copies `count * size_of::()` bytes; source/destination validity, + alignment, and nonoverlap requirements; base case only. +- `https://doc.rust-lang.org/1.80.0/reference/type-layout.html#primitive-data-layout` + — `u8` has size 1. +- `https://doc.rust-lang.org/1.80.0/std/primitive.u8.html#impl-Copy-for-u8` + — `u8` implements `Copy` in the compatibility base case. +- `https://doc.rust-lang.org/1.80.0/std/marker/trait.Copy.html` + — exact copy rather than move semantics for `Copy` values in the base and + 1.80.0 endpoint cases. +- `https://doc.rust-lang.org/1.80.0/std/ptr/fn.read.html` + — reads without moving, leaves source unchanged, and requires read validity, + alignment, and initialization for this non-ZST. +- `https://doc.rust-lang.org/1.80.0/std/primitive.u32.html#impl-Copy-for-u32` + — `u32` implements `Copy` in the 1.80.0 endpoint case. +- `https://doc.rust-lang.org/1.82.0/std/ptr/fn.read.html` + — the same named propositions for the separate endpoint case. +- `https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#impl-Copy-for-u32` + — `u32` implements `Copy` in the 1.82.0 endpoint case. +- `https://doc.rust-lang.org/1.82.0/std/marker/trait.Copy.html` + — exact copy rather than move semantics for `Copy` values in the 1.82.0 + endpoint case. + +The `acknowledge` and compatibility results additionally consume the two exact +accepted propositions in target file `TCB.md`; that file is not Rust authority +and both admissions must remain identified as part of the conditional TCB. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/Q.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/Q.md new file mode 100644 index 0000000000..fcec5e9489 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/Q.md @@ -0,0 +1,76 @@ +# Mode Q Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## Q — Existential refutation and missing unsafe dependency + +Target: supplied Q snapshot. + +Let `C_Q` be Rust 1.80.0 times every target on which the exact source and pinned +dependency resolve times every ordinary profile. + +- `Required(Q-force-sound) = C_Q` times the one well-typed safe call + `force_unreachable()`. +- `Required(Q-delegate-sound) = C_Q` times every call + `delegated_nonzero(value)` whose caller satisfies `value != 0`. + +### Expected results and atoms + +- **Q1 — Domain and claim separation:** State both valid-use domains and audit + the two APIs independently. In particular, the `unsafe fn`'s documented + nonzero precondition restricts its valid uses; it does not restrict the safe + `force_unreachable` API. +- **Q2 — Complete UB certificate:** For `force_unreachable`, give all four + links explicitly or by an equivalently exact chain: + 1. `force_unreachable()` is a well-typed, valid safe call in `Required`; + 2. the only body path reaches the executed + `std::hint::unreachable_unchecked()` call; + 3. that call requires its site not to be reached, and that exact proposition + is false for this call; and + 4. Rust 1.80.0 std documentation says reaching the function is undefined + behavior. +- **Q3 — Existential closure:** Report `Q-force-sound` as **UNSOUND**. The one + complete in-scope witness refutes the universal soundness claim; do not + weaken it to `UNPROVED` because no proof covering every safe call was built. + The witness exists in every `C_Q` configuration, but that stronger fact is + not required for the existential verdict. +- **Q4 — Exact missing premise:** For `delegated_nonzero`, recognize that a + valid caller supplies the dependency API's nonzero caller-side precondition, + but the selected unsafe dependency's body is absent and no accepted + `UNSAFE-DEP` entry establishes that it honors its contract. The exact pin and + caller-facing documentation establish the logical package coordinate and the + propagated obligation, not authenticated implementation identity or + correctness. For the soundness claim actually requested here, the smallest + missing proposition is that every valid nonzero call to the selected + `peer-unsafe-dep` 3.4.5 implementation is free of UB over this exact domain. +- **Q5 — No fabricated witness or silent trust:** Report + `Q-delegate-sound` as **UNPROVED**, not `PROVED` and not `UNSOUND`. From the + submitted packet, claim-level `Covered(Q-delegate-sound)` contains no case: + the dependency-soundness obligation is unresolved for all valid inputs. + There is also no proved execution of the unavailable body, so no UB witness + may be invented. State that recursive source audit or an explicitly accepted + exact `UNSAFE-DEP` entry would be required. + +### Q hard errors + +Any of these is a hard error: + +- calling `force_unreachable` sound or only `UNPROVED` after stating all four + certificate links; +- issuing `UNSOUND` for it without a valid call, reachability, the false exact + safety proposition, and the authoritative UB consequence; +- silently trusting the unavailable unsafe dependency, treating the exact pin + as implementation proof, or reporting `delegated_nonzero` `PROVED`; +- fabricating dependency body behavior or a UB execution and reporting + `delegated_nonzero` `UNSOUND`; or +- treating the unsafe dependency's documented caller precondition as though + violating its prose were itself an executed UB event. + +### Q checked authority + +- `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` + — exact proposition consumed: reaching + `unreachable_unchecked` is undefined behavior; applicability Rust 1.80.0, + without a target/profile qualification relevant to this fixture. + +No Rust authority is asserted for the absent dependency implementation. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/R.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/R.md new file mode 100644 index 0000000000..17b4a752e1 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/R.md @@ -0,0 +1,67 @@ +# Mode R Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## R — abstraction redesign + +### Exact result + +Within the exact target/profile scope declared in `REQUEST.md`, `Required` +includes every safe downstream implementation of `Slot` and safe instantiation +of `increment`, not only the crate-owned `Tail`. A safe +implementation can return 2. Calling `increment::(&mut [0, 0])` then +executes `get_unchecked_mut(2)` on a length-2 slice. The Rust 1.82.0 contract +requires an in-bounds index and says an out-of-bounds call is UB even if the +result is not used. The current safe API is therefore `UNSOUND`. + +The requested behavior needs neither generic pointer/index metadata nor an +unsafe abstraction. A preferred redesign is a nongeneric safe function that +updates `pair[1]` (or an equivalent checked safe specialization). It eliminates +the caller implementation capability and the unsafe block. Making `Slot` +unsafe, adding a prose rule to the safe trait, or preserving the generic +abstraction by default is less parsimonious because no downstream generic use +is required. The proposal does not affect the current verdict and needs a fresh +audit after implementation. + +### Atoms + +- **R1:** Quantify over arbitrary safe `Slot` implementations; explicitly + construct or describe one returning 2 and the safe call using it. +- **R2:** Prove reachability of `get_unchecked_mut(2)`, falsity of its in-bounds + precondition for length 2, and the applicable Rust 1.82 UB consequence. +- **R3:** Report the current safe API `UNSOUND`, independently of design intent + and proposals. +- **R4:** Extract the exact minimum required behavior: wrapping increment of + element 1 for the owned use, with no generic downstream implementation need. +- **R5:** Propose a nongeneric safe specialization that implements the required + wrapping increment of element 1 using checked indexing (or an exactly + behavior-equivalent safe operation), with no caller-controlled + implementation or index capability. +- **R6:** Explain that removing the unpublished generic trait/API is an + authorized contract delta with no promised downstream migration burden. +- **R7:** Keep the redesign conditional and require implementation plus fresh + audit; do not use it to narrow or alter the current `UNSOUND` result. + +Merely making the trait unsafe does not pass R5: it preserves a caller +implementation capability that the supplied requirement expressly does not +need. + +### Hard errors + +Hard error if the report: + +- trusts `Tail` as the only safe implementation; +- treats safe trait prose as a caller obligation; +- calls the current API sound or merely `UNPROVED` after closing the witness; +- certifies an unimplemented redesign; or +- lets design intent narrow the current contract. + +### Authority + +- Rust 1.82.0 slice `get_unchecked_mut`: + `https://doc.rust-lang.org/1.82.0/std/primitive.slice.html#method.get_unchecked_mut` + — the index must be in bounds; out-of-bounds calls are UB even if the + resulting reference is unused. +- Rust 1.82.0 `u32::wrapping_add`: + `https://doc.rust-lang.org/1.82.0/std/primitive.u32.html#method.wrapping_add` + — wrapping modular addition supplies the requested update semantics. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/S.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/S.md new file mode 100644 index 0000000000..b7dd209431 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/S.md @@ -0,0 +1,111 @@ +# Mode S Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## S — Symbolic Interval and Parametric Superset + +### Intended theorem domain + +Let: + +```text +R_S = { r | StableRustRelease(r) and 1.84.0 <= r <= 1.86.0 } +T_S = { x86_64-unknown-linux-gnu, + aarch64-apple-darwin, + wasm32-unknown-unknown } +F_S = { telemetry-off, telemetry-on } +O_S = { None } union { Some(b) | b is any u8 } +``` + +The exact requested case predicate is: + +```text +Required_S(r, t, f, p, d, o) + iff r in R_S and t in T_S and f in F_S and p in P and d in D and o in O_S. +``` + +This predicate is symbolic. It is not the CI matrix or the Cargo minimum. It +contains non-`.0` stable releases such as Rust 1.84.1 and 1.85.1, but a report +may preserve the exact symbolic predicate without enumerating any members. + +Define `Q_Option(r)` to mean that the two required `Option` propositions have +been established for release `r` by either of two admissible bases: + +1. the Rust 1.84.0 base authorities plus accepted entry + `COMPAT-OPTION-184-186` over its exact region; or +2. an exact finite partition that proves the released members of `R_S` are + `{1.84.0, 1.84.1, 1.85.0, 1.85.1, 1.86.0}` and verifies the two exact + versioned pages for every member. + +The intended proof cases are: + +```text +Covered_S = { (r,t,f,p,d,o) | Q_Option(r), and t/f/p/d/o are otherwise arbitrary }. +``` + +The source derivation is parametric in `t`, `f`, `p`, and `d`; it need not and +should not be expanded into their Cartesian product. Applicability still must +establish `R_S subset-of {r | Q_Option(r)}`. Either admissible basis above can +do so. A merely report-authored compatibility proposal, generic stability +assertion, endpoint sampling, or incomplete release partition cannot. + +### Expected verdicts + +- Safe-API soundness over `Required_S`: **PROVED**, either from the exact + finite authoritative partition or relative to `COMPAT-OPTION-184-186`. +- The documented `None -> 0` and `Some(b) -> b` behavior over `Required_S`: + **PROVED** on the same basis and with the TCB qualification whenever that + entry is consumed. +- Configuration closure: **PROVED** on the chosen basis, because + `Required_S subset-of Covered_S`. + +The local proof is: `is_none()` returning true takes the `return 0` branch. If +the unsafe call is reached, `is_none()` returned false; since `Option` has only +`None` and `Some`, the unchanged, still-owned value is `Some`. This discharges +`unwrap_unchecked`'s exact precondition, and its return contract supplies the +contained byte. No target, feature, profile, or debug-assertion fact enters the +derivation. + +### Atoms + +- **S1 — Lossless domain recovery:** States `R_S` as the closed symbolic + stable-release predicate and keeps the three targets and both feature states. + It does not substitute Cargo's lower bound or CI's rows for the policy. Exact + symbolic notation passes without naming any particular patch release. +- **S2 — Premise applicability:** Uses one complete admissible evidence basis: + either verifies the 1.84.0 base, applies only fixture-supplied accepted entry + `COMPAT-OPTION-184-186`, and keeps that admission conspicuous; or proves the + exact five-member release partition and verifies both exact pages for every + member. A report-authored premise, partial partition, or mixed gap does not + pass. +- **S3 — Local safety proof:** Gives the reachability/variant argument that + reaching `unwrap_unchecked` implies the unchanged, still-owned value is + `Some`, thereby discharging the exact unsafe precondition. +- **S4 — Local behavioral proof:** Shows that `None` returns zero and `Some(b)` + returns `b`, using the applicable exact `Option` propositions. +- **S5 — Parametric configuration coverage:** Shows that the local derivation + is independent of target, feature, profile, and debug assertions and covers + their complete product without Cartesian enumeration. +- **S6 — Soundness closure and verdict:** Establishes + `Required_S subset-of Covered_S` for the soundness theorem and reports + soundness `PROVED` over the exact policy domain, conspicuously qualified by + `COMPAT-OPTION-184-186` if and only if that route was consumed. +- **S7 — Behavioral closure and verdict:** Establishes + `Required_S subset-of Covered_S` for the documented result theorem and + reports behavior `PROVED` over the exact policy domain, conspicuously + qualified by `COMPAT-OPTION-184-186` if and only if that route was consumed. + +### Hard-error triggers + +A report commits a hard error if it: + +- claims full closure or an affirmative whole-domain result after contracting + the release predicate to `.0` releases, CI rows, or the Cargo minimum; +- calls endpoints, a toolchain pin, or sparse documentation a proof of every + interior release and then claims `PROVED`; +- silently invents or accepts a report-authored compatibility premise, or hides + the fixture TCB qualification after consuming that entry; +- claims closure while leaving a `Required_S` region outside its proof; +- finds `None` reachable at `unwrap_unchecked`; or +- gives an unqualified `UNSOUND` or `CONTRACT-BROKEN` verdict for the shown + source. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/SCORER.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/SCORER.md new file mode 100644 index 0000000000..9f96351959 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/SCORER.md @@ -0,0 +1,72 @@ +# Frozen Common Blind-Scoring Rules + +Score explicit propositions, not keywords, terminology, organization, or a +preferred proof style. Accept any equivalently exact derivation. Do not infer a +missing material premise from vague shorthand. + +For each report A–J: + +1. Mark every mode atom `PASS` or `FAIL`; every material clause of an atom must + hold for `PASS`. +2. Record every independently applicable hard error. +3. Separately decide proposal laundering, a TCB/authority defect, and a visible + source-scope defect. The runner, not the scorer, decides the mechanical word + budget and records operational scope deviations. +4. Cite compact report evidence for every decision. +5. Record genuine rubric ambiguity and novel findings without creating a new + confirmatory atom. + +`Required` is the exact product of release/configuration scope, valid uses, and +the domain of the theorem being judged. `Covered` contains a case only when +every obligation has a complete applicable derivation. A universal positive +result requires an explicit or readily checkable `Required subset-of Covered` +derivation. An existential `UNSOUND` result instead requires a valid in-scope +safe use, reachability, falsity of the exact safety proposition, and an +applicable authoritative UB consequence. + +Where a mode uses these symbols, `P` ranges over every Cargo profile supported +by the target and `D` ranges over both states of debug assertions. + +A missed atom is not automatically a hard error. Apply a hard error only when +the report makes the listed false affirmative claim or triggers another exact +mode/global rule. Additional correct regional detail is harmless. + +A material Rust premise passes only when the report verifies applicable, +version-matched Rust Reference or standard-library text, or correctly leaves +the proposition unresolved. The URL allowlist grants page identity, not a +proposition. Cargo documentation is not a Rust semantic axiom. A target TCB +entry is not Rust authority; accept it only at its exact accepted identity, +proposition, consumer, and region, with conspicuous qualification where used. + +Mark `proposal_laundering` present exactly when a report uses an unimplemented +proposal as evidence that the supplied current artifact is sound, meets a +current postcondition, or has narrower obligations, or when it certifies the +proposal without implementation and fresh audit. Record G10 and any applicable +mode hard-error ID independently; this flag does not replace them. + +Global hard errors have IDs G1 through G12 in the list order below. A mode's +own hard errors have IDs `H1`, `H2`, and so on in the list order +under that rubric's **Hard errors** heading. Record an exact applicable ID, not +a report-authored category name. Global hard errors are: + +- claiming full `PROVED` after contracting or failing to justify `Required`; +- claiming positive closure without `Required subset-of Covered`; +- inventing policy precedence or calling a conservative audit domain the + project's promise; +- treating CI, endpoints, sparse documentation, a cutoff, or a stability badge + as interval coverage; +- issuing `UNSOUND` without all four existential certificate links; +- issuing only `UNPROVED` after explicitly establishing all four links; +- issuing `CONTRACT-BROKEN` using only an execution that contains UB; +- claiming an observation remains guaranteed because it occurs before UB; +- silently trusting a third-party unsafe implementation or caller-controlled + safe behavior; +- certifying an unimplemented design or using it to narrow the current-artifact + obligation; +- necessarily relying on unchecked, invalid, or inapplicable authority; or +- evidence that the report agent inspected prohibited oracle, sibling, + condition-map, prior-report, or evaluator material. + +Do not identify, cluster, or speculate about generating conditions. Do not +inspect enclosing or sibling directories, other modes, manifests, skill +packages, condition maps, prior scores, or another scorer's output. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/W.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/W.md new file mode 100644 index 0000000000..37a9acfe1a --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/W.md @@ -0,0 +1,66 @@ +# Mode W Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## W — Whole-execution UB and independent behavioral refutation + +Target: supplied W snapshot. + +Let `C_W` be Rust 1.80.0 times every target on which this exact source and the +used Rust 1.80.0 standard-library item exist times every ordinary profile, and +let `I = {0, ..., 255}`. The claims are: + +- `Required(W-sound) = C_W times I`; +- `Required(W-panic) = C_W times {0}`; and +- `Required(W-return) = C_W times I`; its theorem is: for every UB-free defined + execution in this domain, if that execution returns normally, its result + equals its input. + +### Expected results and atoms + +- **W1 — Soundness witness:** `classify(0)` is a valid safe call, takes the + zero arm, evaluates the local statements, and reaches + `unreachable_unchecked`. Its unreachable-site proposition is false; the + applicable 1.80.0 std contract entails UB. Report `W-sound` **UNSOUND**. +- **W2 — Whole-execution classification:** The input-zero execution as a whole + contains UB. Neither its apparent prefix computation nor the source-level + fact that no panic operation appears before the unsafe call remains a + guaranteed observation from a defined execution. It therefore cannot prove + a UB-free failure of the panic postcondition. With no independent UB-free + input-zero execution or equivalent proof, report `W-panic` **UNPROVED**, not + `CONTRACT-BROKEN` and not `PROVED`. Its claim-level `Covered` set is empty on + the submitted evidence. +- **W3 — Independent defined witness:** `classify(1)` takes the `1 => 2` arm, + executes no unsafe operation, returns normally, and returns `2` rather than + the promised input value `1`. That complete execution is UB-free and proves + `W-return` **CONTRACT-BROKEN**. + +Scoring guidance, not a separate atom: keep the three executions/theorems +independent. The input-zero UB witness cannot establish failure of the panic +promise, while the independent, defined input-one failure remains usable and +does not alter the soundness witness. + +### W hard errors + +Any of these is a hard error: + +- calling the safe API sound, reporting only `UNPROVED` after explicitly + establishing every input-zero UB-certificate link, or issuing `UNSOUND` + without those links; +- using the UB-containing input-zero execution, an apparent observation before + its UB, or the absence of an earlier panic to issue `CONTRACT-BROKEN`; +- claiming source observations remain guaranteed merely because they occur + before the unsafe call; +- falsely treating the independent input-one refutation as UB-containing, or + claiming complete behavioral coverage while omitting it. + +### W checked authorities + +- `https://doc.rust-lang.org/1.80.0/std/hint/fn.unreachable_unchecked.html#safety` + — reaching the function is undefined behavior. +- `https://doc.rust-lang.org/1.80.0/reference/behavior-considered-undefined.html` + — controlling Rust 1.80.0 Reference section for what constitutes undefined + behavior. Verify that the final evaluator instructions do not attribute to + this page prose it does not contain; whole-execution non-observability is the + logical consequence of classifying the execution as undefined, not a claimed + quotation from a “time travel” subsection. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/X.md b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/X.md new file mode 100644 index 0000000000..51a1f0c935 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/rubrics/X.md @@ -0,0 +1,145 @@ +# Mode X Frozen Blind-Scoring Rubric + +> **Evaluator-only material. Never expose this file to a report agent.** + +## X — Simultaneous Feature, Target, and Allocator + +### Intended theorem domain + +Let: + +```text +R_X = { Rust 1.85.1 } +T_X = { X = x86_64-unknown-linux-gnu, + A = aarch64-unknown-linux-gnu, + W = wasm32-unknown-unknown } +B_X = { burst-off, burst-on } +L_X = { system, arena } +I_X = { every u8 } +``` + +The exact supported configuration predicate is: + +```text +Config_X(t,b,l) + iff t in T_X and b in B_X and l in L_X and not (t = W and l = arena). + +Required_X = R_X cross Config_X cross P cross D cross I_X. +``` + +In particular, this simultaneous cell is supported: + +```text +Q_X = (target = A, burst = on, allocator = arena). +``` + +The only target/allocator exclusion is `target = W and allocator = arena`, +for either feature state. It is distinct from `Q_X`. + +The build script maps the accepted `FIXTURE_ALLOCATOR` value to exactly one +`fixture_allocator` option; the Rust conditional-compilation rules then select +the corresponding source. For positive soundness bookkeeping, the +implementation closes exactly these call cases: + +```text +Covered_X_sound = { case in Required_X | not Q_X or value != 0 }. +``` + +Outside `Q_X`, the explicit zero check panics for zero; after that check, +`new_unchecked(value)` meets its precondition. Inside `Q_X`, the unchecked +constructor is reached without that check and meets its precondition only for +nonzero values. Consequently `Required_X` is not a subset of +`Covered_X_sound`. + +### Expected verdicts + +- Safe-API soundness over `Required_X`: **UNSOUND relative to accepted + `BUILD-MAP-X`**. +- Whole-domain postcondition “zero panics”: **UNPROVED**, not + `CONTRACT-BROKEN`, from the known counterexample, because that execution + contains UB; the configuration classification is relative to `BUILD-MAP-X`. +- Soundness outside `Q_X`, and for nonzero inputs inside `Q_X`: **PROVED + relative to `BUILD-MAP-X`** with the version-matched contracts. +- The documented zero-input panic guarantee outside `Q_X`: **PROVED relative + to `BUILD-MAP-X`**. +- The wasm32/arena pair: genuinely excluded and rejected; it is not the UB + witness. Effective rejection is **PROVED relative to `BUILD-MAP-X`**. + +The closing witness is the fully safe call `lane_id(0)` on Rust 1.85.1 for +`aarch64-unknown-linux-gnu`, with `burst` enabled and the accepted `arena` +allocator selection. The build output and all three true `cfg` conjuncts make +the unsafe branch reachable. It calls `NonZeroU8::new_unchecked(0)`, whose +exact safety requirement is false and whose applicable documentation states +the UB consequence. One supported witness closes `UNSOUND`. + +### Atoms + +- **X1 — Complete cross-axis domain:** Recovers the target, feature, and + allocator axes together; includes `Q_X`; and records only wasm32/arena as the + policy exclusion. +- **X2 — Selector partition:** Recovers the complete environment-input + partition: omitted, Unicode `system`, Unicode `arena`, every other Unicode + value, and every non-Unicode value. +- **X3 — Accepted selector mapping:** Shows that omitted and explicit `system` + select `system`, while explicit `arena` selects `arena`, on build attempts + whose directive writes succeed. +- **X4 — Rejected selector behavior:** On attempts where the preceding + directive writes succeed, shows that every other Unicode value and every + non-Unicode value reaches a panic arm and, under `BUILD-MAP-X`, produces no + library compilation. It distinguishes that policy rejection from an earlier + infrastructure write failure, which also produces no compilation but never + reaches selector handling. +- **X5 — Allocator-cfg cardinality:** Shows that every successful accepted + selector path emits exactly one `fixture_allocator` cfg and that its value is + the value selected in X3. +- **X6 — Check-cfg directive:** After the preceding rerun write succeeds, shows + that the script attempts the check-cfg write before selector handling and + that, when this write succeeds, it registers exactly `system` and `arena` as + expected values without selecting either value. Failure of either write + follows the infrastructure no-compilation branch. +- **X7 — Rerun directive:** Shows that the script unconditionally attempts the + environment-change rerun write and that, when the write succeeds, + `BUILD-MAP-X` causes Cargo to rerun the script when `FIXTURE_ALLOCATOR` + changes. A failed write follows the infrastructure no-compilation branch. +- **X8 — Generated-configuration reachability:** Follows accepted `arena` + selection through `build.rs` to `fixture_allocator="arena"` and then applies + the feature, target, and allocator cfg conjuncts to reach `Q_X`. It does not + audit only handwritten library source in isolation. +- **X9 — Existential UB certificate:** Gives the supported safe + `lane_id(0)` call in `Q_X`, reaches `new_unchecked(0)`, states the exact + false nonzero proposition and applicable UB consequence, and reports + `UNSOUND`. +- **X10 — Quantifier-sensitive behavior:** Calls the universal zero-input panic + guarantee `UNPROVED`, not `CONTRACT-BROKEN`, because the known failing + execution has UB. +- **X11 — Remainder soundness:** Proves soundness outside `Q_X` by following the + explicit zero guard and applying `new_unchecked` only after it establishes a + nonzero value, and proves nonzero inputs sound inside `Q_X`, relative to the + exact build mapping. +- **X12 — Regional panic theorem:** Proves that zero takes the explicit + `panic!` branch in every configuration outside `Q_X`, relative to the exact + build mapping. It asserts no undocumented nonzero return-value postcondition. +- **X13 — Genuine exclusion:** Shows both that policy excludes wasm32/arena and + that the active `compile_error!` rejects that pair after allocator cfg + generation under `BUILD-MAP-X`. It does not use this excluded pair as the + witness or generalize the exclusion to aarch64/arena. + +### Hard-error triggers + +A report commits a hard error if it: + +- omits feature, target, or allocator and then uses the contracted conjunction + to claim full-domain closure/soundness, or treats the supported `Q_X` cell as + excluded; +- ignores build-script output and consequently declares the unsafe branch + unreachable; +- invents, hides, or widens a build-tool/target mapping beyond accepted + `BUILD-MAP-X`; +- uses the excluded wasm32/arena pair as a valid soundness counterexample; +- calls the current safe API sound or stops at `UNPROVED` despite establishing + the complete supported `lane_id(0)` UB witness; +- calls the zero-panic promise `CONTRACT-BROKEN` based on the UB-containing + execution, or claims observations before/after that UB remain guaranteed; +- claims the policy exclusion alone makes a reachable safe API sound without + checking effective rejection; or +- certifies an unimplemented repair instead of the supplied source snapshot. diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/adjudication.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/adjudication.schema.json new file mode 100644 index 0000000000..a9ceffde2b --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/adjudication.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-blind-adjudication-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "decisions", "ambiguities"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "decisions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "field", "decision", "evidence"], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "field": { + "type": "string", + "pattern": "^(atom:[SCXQWMRK][1-9][0-9]*|hard_error:(G[1-9][0-9]*|[SCXQWMRK]H[1-9][0-9]*)|proposal_laundering|tcb_authority_defect|visible_scope_defect|novel:s[12]:N[1-9][0-9]*)$" + }, + "decision": { + "enum": ["PASS", "FAIL", "PRESENT", "ABSENT"] + }, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + }, + "ambiguities": { + "type": "array", + "items": {"type": "string", "pattern": "\\S"} + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/disagreements.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/disagreements.schema.json new file mode 100644 index 0000000000..ae7b277ff5 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/disagreements.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-v3-targeted-disagreements-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "cells"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "cells": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "field", "s1", "s2"], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "field": {"type": "string", "minLength": 1}, + "s1": {"$ref": "#/$defs/position"}, + "s2": {"$ref": "#/$defs/position"} + } + } + } + }, + "$defs": { + "position": { + "type": "object", + "additionalProperties": false, + "required": ["decision", "evidence"], + "properties": { + "decision": {"enum": ["PASS", "FAIL", "PRESENT", "ABSENT"]}, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/event.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/event.schema.json new file mode 100644 index 0000000000..a49542dff4 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/event.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-v3-targeted-event-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "sequence", "previous_event_sha256", "time_utc", "phase", "event", "details"], + "properties": { + "schema_version": {"const": 1}, + "sequence": {"type": "integer", "minimum": 1}, + "previous_event_sha256": { + "oneOf": [ + {"type": "null"}, + {"type": "string", "pattern": "^[0-9a-f]{64}$"} + ] + }, + "time_utc": {"type": "string", "format": "date-time"}, + "phase": { + "enum": ["freeze", "collection", "scoring", "adjudication", "unblinding", "result"] + }, + "event": {"type": "string", "minLength": 1}, + "run_id": {"type": "string"}, + "attempt": {"type": "integer", "minimum": 1}, + "agent_id": {"type": "string"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "details": {"type": "object"} + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/final.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/final.schema.json new file mode 100644 index 0000000000..53d7343921 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/final.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-v3-targeted-final-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "reports", "confirmed_novel_findings"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "reports": { + "type": "array", + "minItems": 10, + "maxItems": 10, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "atoms", "hard_errors", "proposal_laundering", "tcb_authority_defect", "visible_scope_defect"], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "atoms": {"type": "object"}, + "hard_errors": {"type": "array", "items": {"type": "string"}}, + "proposal_laundering": {"type": "boolean"}, + "tcb_authority_defect": {"type": "boolean"}, + "visible_scope_defect": {"type": "boolean"} + } + } + }, + "confirmed_novel_findings": { + "type": "array", + "items": {"type": "object"} + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/score.schema.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/score.schema.json new file mode 100644 index 0000000000..b9509706d0 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/freeze/schemas/score.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "unsafe-rust-blind-score-v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "mode", "scorer_id", "reports", "ambiguities"], + "properties": { + "schema_version": {"const": 1}, + "mode": {"enum": ["S", "C", "X", "Q", "W", "M", "R", "K"]}, + "scorer_id": {"enum": ["s1", "s2"]}, + "reports": { + "type": "array", + "minItems": 10, + "maxItems": 10, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "label", + "atoms", + "hard_errors", + "proposal_laundering", + "tcb_authority_defect", + "visible_scope_defect", + "novel_findings" + ], + "properties": { + "label": {"type": "string", "pattern": "^[A-J]$"}, + "atoms": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "decision", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^[SCXQWMRK][1-9][0-9]*$"}, + "decision": {"enum": ["PASS", "FAIL"]}, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + }, + "hard_errors": { + "type": "array", + "items": {"$ref": "#/$defs/hard_error"} + }, + "proposal_laundering": {"$ref": "#/$defs/flag"}, + "tcb_authority_defect": {"$ref": "#/$defs/flag"}, + "visible_scope_defect": {"$ref": "#/$defs/flag"}, + "novel_findings": { + "type": "array", + "items": {"$ref": "#/$defs/novel_finding"} + } + } + } + }, + "ambiguities": { + "type": "array", + "items": {"type": "string", "pattern": "\\S"} + } + }, + "$defs": { + "hard_error": { + "type": "object", + "additionalProperties": false, + "required": ["id", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^(G[1-9][0-9]*|[SCXQWMRK]H[1-9][0-9]*)$"}, + "evidence": {"type": "string", "pattern": "\\S"} + } + }, + "novel_finding": { + "type": "object", + "additionalProperties": false, + "required": ["id", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^N[1-9][0-9]*$"}, + "evidence": {"type": "string", "pattern": "\\S"} + } + }, + "flag": { + "type": "object", + "additionalProperties": false, + "required": ["present", "evidence"], + "properties": { + "present": {"type": "boolean"}, + "evidence": {"type": "string", "pattern": "\\S"} + } + } + } +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/operations.lock b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/operations.lock new file mode 100644 index 0000000000..e69de29bb2 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/prepare.py b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/prepare.py new file mode 100644 index 0000000000..23a3d8b343 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/prepare.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Generate and verify the frozen V3 targeted-evaluation artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + + +RUN = Path(__file__).resolve().parent +FREEZE = RUN / "freeze" +SEALED = RUN / "sealed" +EVALS = RUN.parent.parent + +MODES = ("S", "C", "X", "Q", "W", "M", "R", "K") +CONDITIONS = ("v3", "v2") +REPLICATES = range(1, 6) +ATOM_COUNTS = {"S": 7, "C": 6, "X": 13, "Q": 5, "W": 3, "M": 11, "R": 7, "K": 8} +WORD_CAPS = {"S": 1800, "C": 1800, "X": 2400, "Q": 1800, "W": 1800, "M": 1800, "R": 1800, "K": 2200} + +PACKAGES = { + "v3": { + "path": "frozen-packages/668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf", + "tree": "668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf", + "skill": "0e23f7747cc63014bade7543efaf745e7e9a7e5d6dee2a48c602ef7a3eba091e", + }, + "v2": { + "path": "frozen-packages/40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897", + "tree": "40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897", + "skill": "a0a75ef8a14497aa78b50b459981097ee99605c57fec95c637cf59aaa20fe766", + }, +} + +TARGETS = { + "S": ("s_symbolic", "28ecc523e15b914a187814ab2752c0d85996948a552c62f970d8484bc6ed467a"), + "C": ("c_conflict", "065c3cfc032af93e7576e17e49322826c4a379a707870175b16c2663d1e8e4e0"), + "X": ("x_cross", "25b4efef689601f3b5983bf6b914bd367153d0b21a6f1f4d40de50c5d412afa7"), + "Q": ("q_quantifiers", "c0a4c43373a159cb38d08724af8b02187b249ab6a73f7e1b10b2276f38b0cb5a"), + "W": ("w_whole_execution", "b27b95fbc9ffa9d335bb6b4614a9f227a5798122fca79b42cf53c9b106a6aff6"), + "M": ("m_multirelease", "b269cf068196d1c06b87be6bcded494827e7ac8a2cb6debf1eb0f0f5d0388479"), + "R": ("r_redesign", "6d1a41909b012484d71f94d194071a7ebbb6773b7596db88127ca2a195b70ffc"), + "K": ("k_regression", "ca272e524b36a892e25f6169631a184ff764026451eff2f6ac4ab8e9d5e87ea2"), +} + +RUBRIC_RANGES = { + "S": ("domain.md", "## S —", "## C —"), + "C": ("domain.md", "## C —", "## X —"), + "X": ("domain.md", "## X —", "## Exact Authority"), + "Q": ("verdict.md", "## Q —", "## W —"), + "W": ("verdict.md", "## W —", "## M —"), + "M": ("verdict.md", "## M —", None), + "R": ("controls.md", "## R —", "## K —"), + "K": ("controls.md", "## K —", None), +} + +AUTHORITY_RANGES = { + "S": ("domain.md", "### S authorities", "### C authorities"), + "C": ("domain.md", "### C authorities", "### X authorities"), + "X": ("domain.md", "### X authorities", None), + "Q": ("verdict.md", "## Q —", "## W —"), + "W": ("verdict.md", "## W —", "## M —"), + "M": ("verdict.md", "## M —", None), + "R": ("controls.md", "## R —", "## K —"), + "K": ("controls.md", "## K —", None), +} + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def keyed(tag: str, seed: str, value: str) -> str: + return sha256(tag.encode() + b"\0" + bytes.fromhex(seed) + b"\0" + value.encode()) + + +def read_seeds() -> dict[str, str]: + data = json.loads((SEALED / "seeds.json").read_text()) + expected = {"condition", "schedule", "blind", "presentation", "scorer"} + if set(data) != expected: + raise ValueError("unexpected seed keys") + for name, value in data.items(): + if not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError(f"invalid {name} seed") + if value == "0" * 64: + raise ValueError(f"zero {name} seed") + if len(set(data.values())) != len(data): + raise ValueError("randomization seeds are not distinct") + return data + + +def extract_rubric(mode: str) -> str: + filename, start_marker, end_marker = RUBRIC_RANGES[mode] + source = (FREEZE / "oracle" / filename).read_text() + start = source.index(start_marker) + end = source.index(end_marker, start + len(start_marker)) if end_marker else len(source) + body = source[start:end].rstrip() + "\n" + atoms = re.findall(rf"^- \*\*{mode}[0-9]+\b", body, flags=re.MULTILINE) + if len(atoms) != ATOM_COUNTS[mode]: + raise ValueError(f"{mode}: expected {ATOM_COUNTS[mode]} atoms, found {len(atoms)}") + return ( + f"# Mode {mode} Frozen Blind-Scoring Rubric\n\n" + "> **Evaluator-only material. Never expose this file to a report agent.**\n\n" + + body + ) + + +def generated_files(seeds: dict[str, str]) -> dict[Path, str]: + outputs: dict[Path, str] = {} + + for mode in MODES: + outputs[FREEZE / "rubrics" / f"{mode}.md"] = extract_rubric(mode) + + condition_order = sorted(CONDITIONS, key=lambda role: (keyed("condition-v1", seeds["condition"], role), role)) + condition_labels = {role: f"c{index}" for index, role in enumerate(condition_order)} + + mode_order = sorted(MODES, key=lambda mode: (keyed("mode-label-v1", seeds["schedule"], mode), mode)) + mode_labels = {mode: f"m{index}" for index, mode in enumerate(mode_order)} + + condition_rows = ["condition_label\trole\tpackage_path\ttree_sha256\tskill_sha256"] + for role in condition_order: + package = PACKAGES[role] + condition_rows.append( + "\t".join((condition_labels[role], role, package["path"], package["tree"], package["skill"])) + ) + outputs[SEALED / "condition-map.tsv"] = "\n".join(condition_rows) + "\n" + + target_rows = ["target_label\tmode\tsource_path\ttree_sha256\tword_cap"] + for mode in mode_order: + directory, digest = TARGETS[mode] + target_rows.append( + f"{mode_labels[mode]}\t{mode}\tfixtures/v3-targeted/{directory}\t{digest}\t{WORD_CAPS[mode]}" + ) + outputs[SEALED / "target-map.tsv"] = "\n".join(target_rows) + "\n" + + wave_order = sorted(REPLICATES, key=lambda rep: (keyed("wave-v1", seeds["schedule"], str(rep)), rep)) + schedule_rows = ["run_id\tcell_id\twave\ttarget_label\tcondition_label\treplicate"] + schedule: list[tuple[str, str, int, str, str, int, str, str]] = [] + run_number = 1 + for wave_index, replicate in enumerate(wave_order, start=1): + cells = [(mode, role, replicate) for mode in MODES for role in CONDITIONS] + cells.sort( + key=lambda cell: ( + keyed("schedule-v1", seeds["schedule"], f"{cell[0]}|{cell[1]}|{cell[2]}"), + cell, + ) + ) + for mode, role, rep in cells: + canonical = f"{mode}|{role}|{rep}" + cell_id = keyed("cell-v1", seeds["schedule"], canonical)[:32] + run_id = f"r{run_number:03d}" + schedule_rows.append( + f"{run_id}\t{cell_id}\t{wave_index}\t{mode_labels[mode]}\t{condition_labels[role]}\t{rep}" + ) + schedule.append((run_id, cell_id, wave_index, mode, role, rep, mode_labels[mode], condition_labels[role])) + run_number += 1 + if len(schedule) != 80 or len({row[1] for row in schedule}) != 80: + raise ValueError("schedule does not contain 80 unique cells") + outputs[SEALED / "launch-schedule.tsv"] = "\n".join(schedule_rows) + "\n" + + blind_rows = ["mode\tlabel\trun_id"] + for mode in MODES: + run_ids = [row[0] for row in schedule if row[3] == mode] + run_ids.sort(key=lambda run_id: (keyed("blind-v1", seeds["blind"], f"{mode}|{run_id}"), run_id)) + for index, run_id in enumerate(run_ids): + blind_rows.append(f"{mode}\t{chr(ord('A') + index)}\t{run_id}") + outputs[SEALED / "blind-map.tsv"] = "\n".join(blind_rows) + "\n" + + presentation_rows = ["claim\tlabels_in_order"] + for mode in MODES: + for scorer in ("s1", "s2"): + labels = [chr(ord("A") + index) for index in range(10)] + labels.sort( + key=lambda label: ( + keyed("presentation-v1", seeds["presentation"], f"{mode}|{scorer}|{label}"), + label, + ) + ) + presentation_rows.append(f"{mode}-{scorer}\t{','.join(labels)}") + outputs[SEALED / "presentation-orders.tsv"] = "\n".join(presentation_rows) + "\n" + + claims = [f"{mode}-{scorer}" for mode in MODES for scorer in ("s1", "s2")] + claims.sort(key=lambda claim: (keyed("scorer-v1", seeds["scorer"], claim), claim)) + outputs[SEALED / "scoring-schedule.tsv"] = "claim\n" + "\n".join(claims) + "\n" + + commitments = { + "schema_version": 1, + "algorithm": "sha256(tag_utf8 || NUL || seed_bytes)", + "commitments": { + name: sha256(f"{name}-v1".encode() + b"\0" + bytes.fromhex(seed)) + for name, seed in sorted(seeds.items()) + }, + } + outputs[FREEZE / "randomization" / "commitments.json"] = json.dumps(commitments, indent=2, sort_keys=True) + "\n" + return outputs + + +def validate_allowlists() -> None: + for mode in MODES: + path = FREEZE / "allowlists" / f"{mode}.txt" + lines = path.read_text().splitlines() + if not lines or len(lines) != len(set(lines)): + raise ValueError(f"{mode}: empty or duplicate allowlist") + if any(not re.fullmatch(r"https://doc\.rust-lang\.org/\S+", line) for line in lines): + raise ValueError(f"{mode}: allowlist is not URL-only") + filename, start_marker, end_marker = AUTHORITY_RANGES[mode] + source = (FREEZE / "oracle" / filename).read_text() + start = source.index(start_marker) + end = source.index(end_marker, start + len(start_marker)) if end_marker else len(source) + oracle_urls = re.findall(r"https://doc\.rust-lang\.org/[^`)\s]+", source[start:end]) + if lines != oracle_urls: + raise ValueError(f"{mode}: allowlist differs from canonical oracle URL order") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--write", action="store_true", help="write generated artifacts") + args = parser.parse_args() + + if args.write and (FREEZE / "LOCK.json").exists(): + raise SystemExit("refusing to rewrite generated artifacts after freeze lock") + + validate_allowlists() + outputs = generated_files(read_seeds()) + mismatches: list[str] = [] + for path, expected in outputs.items(): + if args.write: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(expected) + elif not path.exists() or path.read_text() != expected: + mismatches.append(str(path.relative_to(RUN))) + if mismatches: + raise SystemExit("generated artifact mismatch:\n" + "\n".join(mismatches)) + print(f"validated {len(outputs)} generated artifacts") + + +if __name__ == "__main__": + main() diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/protocol.py b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/protocol.py new file mode 100644 index 0000000000..f03eeb440c --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/protocol.py @@ -0,0 +1,3603 @@ +#!/usr/bin/env python3 +"""Frozen mechanics for the V3 targeted evaluation. + +This program validates immutable inputs, prepares neutral report cells, builds +blind packets, reconciles dual scores, and aggregates only after adjudication. +It never runs or builds a target. +""" + +from __future__ import annotations + +import argparse +import csv +import fcntl +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import io +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import prepare + + +RUN = Path(__file__).resolve().parent +FREEZE = RUN / "freeze" +SEALED = RUN / "sealed" +COLLECTION = RUN / "collection" +SCORING = RUN / "scoring" +RESULTS = RUN / "results" +EVALS = RUN.parent.parent +MODES = prepare.MODES +LABELS = tuple(chr(ord("A") + index) for index in range(10)) +SCORERS = ("s1", "s2") +GLOBAL_HARD_ERROR_COUNT = 12 +INFRA_FAILURE_CODES = { + "SERVICE_ERROR_BEFORE_OUTPUT", + "ORCHESTRATOR_TOOL_FAILURE", + "FILESYSTEM_FAILURE", +} +TERMINAL_REPORT_FAILURE_CODES = { + "REFUSAL", + "TIMEOUT_AFTER_WORK", + "INVALID_OUTPUT", + "SEMANTIC_NONCOMPLETION", +} +FILE_MANIFEST = FREEZE / "file-manifest.sha256" +LOCK = FREEZE / "LOCK.json" +OPERATION_LOCK = RUN / "operations.lock" +EVENT_PHASES = {"freeze", "collection", "scoring", "adjudication", "unblinding", "result"} +_AUTHENTICATED_FILE_DIGESTS: dict[str, str] | None = None + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be a positive integer") + return parsed + + +def acquire_operation_lock() -> Any: + OPERATION_LOCK.parent.mkdir(parents=True, exist_ok=True) + handle = OPERATION_LOCK.open("a+", encoding="utf-8") + fcntl.flock(handle, fcntl.LOCK_EX) + return handle + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + return sha256_bytes(path.read_bytes()) + + +def json_dump(value: Any) -> str: + return json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def is_nonblank_string(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def read_tsv(path: Path) -> list[dict[str, str]]: + with path.open(newline="") as file: + return list(csv.DictReader(file, dialect="excel-tab")) + + +def read_frozen_tsv(path: Path) -> list[dict[str, str]]: + data = read_frozen_bytes(path) + return list(csv.DictReader(io.StringIO(data.decode()), dialect="excel-tab")) + + +def read_frozen_bytes(path: Path) -> bytes: + data = path.read_bytes() + if FILE_MANIFEST.exists() and sha256_bytes(data) != frozen_file_digest(path): + raise ValueError(f"frozen input changed while being read: {path.relative_to(RUN)}") + return data + + +def read_frozen_text(path: Path) -> str: + return read_frozen_bytes(path).decode() + + +def write_once(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8", newline="") as file: + file.write(content) + file.flush() + os.fsync(file.fileno()) + + +def write_bytes_once(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as file: + file.write(content) + file.flush() + os.fsync(file.fileno()) + + +def load_schedule() -> dict[str, dict[str, str]]: + rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + return {row["run_id"]: row for row in rows} + + +def load_condition_map() -> dict[str, dict[str, str]]: + return { + row["condition_label"]: row + for row in read_frozen_tsv(SEALED / "condition-map.tsv") + } + + +def load_target_map() -> dict[str, dict[str, str]]: + return { + row["target_label"]: row for row in read_frozen_tsv(SEALED / "target-map.tsv") + } + + +def load_blind_map() -> dict[str, dict[str, str]]: + by_mode: dict[str, dict[str, str]] = {mode: {} for mode in MODES} + for row in read_frozen_tsv(SEALED / "blind-map.tsv"): + by_mode[row["mode"]][row["label"]] = row["run_id"] + return by_mode + + +def load_frozen_seeds() -> dict[str, str]: + data = json.loads(read_frozen_text(SEALED / "seeds.json")) + expected = {"condition", "schedule", "blind", "presentation", "scorer"} + if not isinstance(data, dict) or set(data) != expected: + raise ValueError("unexpected frozen seed keys") + if any( + not isinstance(value, str) + or not re.fullmatch(r"[0-9a-f]{64}", value) + or value == "0" * 64 + for value in data.values() + ): + raise ValueError("invalid frozen randomization seed") + if len(set(data.values())) != len(data): + raise ValueError("frozen randomization seeds are not distinct") + return data + + +def mode_for_run(run_id: str) -> str: + row = load_schedule()[run_id] + return load_target_map()[row["target_label"]]["mode"] + + +def atom_ids(mode: str) -> tuple[str, ...]: + text = read_frozen_text(FREEZE / "rubrics" / f"{mode}.md") + atoms = tuple(re.findall(rf"^- \*\*({mode}[1-9][0-9]*)\b", text, flags=re.MULTILINE)) + if len(atoms) != prepare.ATOM_COUNTS[mode] or len(set(atoms)) != len(atoms): + raise ValueError(f"invalid atom inventory for mode {mode}: {atoms}") + return atoms + + +def mode_hard_error_ids(mode: str) -> tuple[str, ...]: + text = read_frozen_text(FREEZE / "rubrics" / f"{mode}.md") + match = re.search( + r"^#{2,6}[^\n]*hard[^\n]*\n(.*?)(?=^#{1,6} |\Z)", + text, + flags=re.MULTILINE | re.DOTALL | re.IGNORECASE, + ) + if not match: + raise ValueError(f"missing hard-error section for mode {mode}") + body = match.group(1) + count = len(re.findall(r"^- ", body, flags=re.MULTILINE)) + if not count: + raise ValueError(f"empty hard-error section for mode {mode}") + return tuple(f"{mode}H{index}" for index in range(1, count + 1)) + + +def hard_error_ids(mode: str) -> tuple[str, ...]: + return tuple(f"G{index}" for index in range(1, GLOBAL_HARD_ERROR_COUNT + 1)) + mode_hard_error_ids(mode) + + +def tar_tree_digest(path: Path) -> str: + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"not a directory: {path}") + if any(item.is_symlink() for item in path.rglob("*")): + raise ValueError(f"symlink prohibited in frozen tree: {path}") + command = [ + "tar", + "--sort=name", + "--mtime=@0", + "--owner=0", + "--group=0", + "--numeric-owner", + "-C", + str(path), + "-cf", + "-", + ".", + ] + completed = subprocess.run(command, check=True, stdout=subprocess.PIPE) + return sha256_bytes(completed.stdout) + + +def byte_tree_digest(path: Path) -> str: + if path.is_symlink() or not path.is_dir(): + raise ValueError(f"byte-tree root is not a real directory: {path}") + records: list[bytes] = [] + for item in sorted(path.rglob("*"), key=lambda value: value.relative_to(path).as_posix()): + relative = item.relative_to(path).as_posix() + if item.is_symlink() or not (item.is_dir() or item.is_file()): + raise ValueError(f"unsupported runtime entry: {item}") + if item.is_dir(): + records.append(f"d\0{relative}\n".encode()) + else: + data = item.read_bytes() + records.append( + f"f\0{relative}\0{len(data)}\0{sha256_bytes(data)}\n".encode() + ) + return sha256_bytes(b"".join(records)) + + +def lock_input_paths() -> list[Path]: + paths: list[Path] = [] + for name in ("prepare.py", "fetch_authority.py", "protocol.py"): + paths.append(RUN / name) + for root in (FREEZE, SEALED): + for path in root.rglob("*"): + if not path.is_file(): + continue + if path in {FILE_MANIFEST, LOCK} or "__pycache__" in path.parts: + continue + paths.append(path) + paths.sort(key=lambda path: path.relative_to(RUN).as_posix()) + if any(path.is_symlink() for path in paths): + raise ValueError("symlinks are prohibited in lock inputs") + return paths + + +def render_file_manifest() -> str: + return "".join( + f"{sha256_file(path)} {path.relative_to(RUN).as_posix()}\n" + for path in lock_input_paths() + ) + + +def verify_file_manifest() -> None: + if not FILE_MANIFEST.exists(): + raise ValueError("missing file-manifest.sha256") + expected = render_file_manifest() + actual = FILE_MANIFEST.read_text() + if actual != expected: + raise ValueError("freeze file manifest does not match current inputs") + + +def parse_file_manifest(data: bytes) -> dict[str, str]: + entries: dict[str, str] = {} + for line in data.decode().splitlines(): + digest, separator, name = line.partition(" ") + if ( + not separator + or not name + or name in entries + or not re.fullmatch(r"[0-9a-f]{64}", digest) + ): + raise ValueError("invalid frozen file-manifest row") + entries[name] = digest + return entries + + +def frozen_file_digest(path: Path) -> str: + relative = path.relative_to(RUN).as_posix() + if _AUTHENTICATED_FILE_DIGESTS is not None: + entries = _AUTHENTICATED_FILE_DIGESTS + else: + data = FILE_MANIFEST.read_bytes() + if LOCK.exists(): + lock = json.loads(LOCK.read_text()) + if lock.get("file_manifest_sha256") != sha256_bytes(data): + raise ValueError("file manifest is not authenticated by LOCK.json") + entries = parse_file_manifest(data) + if relative not in entries: + raise ValueError(f"path is not pinned by the freeze manifest: {relative}") + return entries[relative] + + +def validate_static(require_lock: bool, *, announce: bool = True) -> None: + global _AUTHENTICATED_FILE_DIGESTS + prepare.validate_allowlists() + generated = prepare.generated_files(prepare.read_seeds()) + for path, expected in generated.items(): + if not path.exists() or read_frozen_text(path) != expected: + raise ValueError(f"generated artifact mismatch: {path.relative_to(RUN)}") + + for role, package in prepare.PACKAGES.items(): + path = EVALS / package["path"] + actual = tar_tree_digest(path) + if actual != package["tree"]: + raise ValueError(f"{role} package digest mismatch: {actual}") + skill = sha256_file(path / "SKILL.md") + if skill != package["skill"]: + raise ValueError(f"{role} SKILL.md digest mismatch: {skill}") + for mode, (directory, expected) in prepare.TARGETS.items(): + actual = tar_tree_digest(EVALS / "fixtures" / "v3-targeted" / directory) + if actual != expected: + raise ValueError(f"{mode} target digest mismatch: {actual}") + + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + if len(schedule_rows) != 80: + raise ValueError(f"expected 80 schedule rows, found {len(schedule_rows)}") + if len({row["run_id"] for row in schedule_rows}) != 80: + raise ValueError("duplicate run ID") + if len({row["cell_id"] for row in schedule_rows}) != 80: + raise ValueError("duplicate cell ID") + target_map = load_target_map() + condition_map = load_condition_map() + wave_counts: Counter[tuple[str, str, str]] = Counter() + for row in schedule_rows: + mode = target_map[row["target_label"]]["mode"] + role = condition_map[row["condition_label"]]["role"] + wave_counts[(row["wave"], mode, role)] += 1 + expected_wave_counts = Counter( + (str(wave), mode, role) + for wave in range(1, 6) + for mode in MODES + for role in prepare.CONDITIONS + ) + if wave_counts != expected_wave_counts: + raise ValueError("schedule is not five complete balanced waves") + + blind = load_blind_map() + mapped_runs: list[str] = [] + for mode in MODES: + if tuple(sorted(blind[mode])) != LABELS: + raise ValueError(f"{mode} blind labels are incomplete") + for run_id in blind[mode].values(): + if mode_for_run(run_id) != mode: + raise ValueError(f"blind map crosses modes: {mode} {run_id}") + mapped_runs.append(run_id) + atom_ids(mode) + mode_hard_error_ids(mode) + if Counter(mapped_runs) != Counter(row["run_id"] for row in schedule_rows): + raise ValueError("blind map is not a bijection over scheduled runs") + + presentations = read_frozen_tsv(SEALED / "presentation-orders.tsv") + if {row["claim"] for row in presentations} != { + f"{mode}-{scorer}" for mode in MODES for scorer in SCORERS + }: + raise ValueError("presentation claims are incomplete") + for row in presentations: + if tuple(sorted(row["labels_in_order"].split(","))) != LABELS: + raise ValueError(f"invalid presentation order: {row['claim']}") + + for schema in (FREEZE / "schemas").glob("*.json"): + json.loads(read_frozen_text(schema)) + + authority = FREEZE / "authority-manifest.tsv" + if not authority.exists(): + raise ValueError("missing authority-manifest.tsv") + authority_rows = read_frozen_tsv(authority) + allowlist_pairs = [ + (mode, url) + for mode in MODES + for url in read_frozen_text(FREEZE / "allowlists" / f"{mode}.txt").splitlines() + ] + if [(row["mode"], row["requested_url"]) for row in authority_rows] != allowlist_pairs: + raise ValueError("authority manifest does not match exact allowlist sequence") + if any( + row["status"] != "200" + or row["fragment_found"] != "true" + or not re.fullmatch(r"[0-9a-f]{64}", row["sha256"]) + for row in authority_rows + ): + raise ValueError("authority manifest has an invalid retrieval record") + validate_event_ledger() + validate_preserved_artifacts() + + if require_lock: + verify_file_manifest() + lock = json.loads(LOCK.read_text()) + if set(lock) != { + "schema_version", + "status", + "file_manifest_sha256", + "review_signoffs", + "reports_collected_before_lock", + "locked_utc", + }: + raise ValueError("freeze lock has unexpected fields") + if type(lock.get("schema_version")) is not int or lock["schema_version"] != 1 or lock.get("status") != "FROZEN": + raise ValueError("invalid freeze lock") + locked_utc = lock.get("locked_utc") + if not isinstance(locked_utc, str): + raise ValueError("invalid freeze lock timestamp") + try: + locked_time = datetime.fromisoformat(locked_utc.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("invalid freeze lock timestamp") from error + if locked_time.tzinfo is None: + raise ValueError("freeze lock timestamp lacks timezone") + root = sha256_file(FILE_MANIFEST) + if lock.get("file_manifest_sha256") != root: + raise ValueError("freeze lock root does not match file manifest") + signoffs = lock.get("review_signoffs", []) + if not isinstance(signoffs, list) or len(signoffs) < 2 or any( + not isinstance(signoff, dict) for signoff in signoffs + ): + raise ValueError("freeze lock lacks two review signoffs") + reviewer_ids = [signoff.get("reviewer_id") for signoff in signoffs] + if any( + not isinstance(reviewer_id, str) + or not reviewer_id + or reviewer_id != reviewer_id.strip() + for reviewer_id in reviewer_ids + ): + raise ValueError("freeze lock has an empty reviewer ID") + if len(set(reviewer_ids)) != len(signoffs): + raise ValueError("freeze lock reviewer IDs are not distinct") + for signoff in signoffs: + if set(signoff) != { + "reviewer_id", + "verdict", + "file_manifest_sha256", + "scope", + "reviewed_utc", + }: + raise ValueError("freeze review signoff has unexpected fields") + scope = signoff.get("scope") + reviewed_utc = signoff.get("reviewed_utc") + if ( + signoff.get("verdict") != "PASS/FREEZE" + or signoff.get("file_manifest_sha256") != root + or not isinstance(scope, str) + or not scope.strip() + or scope != scope.strip() + or not isinstance(reviewed_utc, str) + or not reviewed_utc.strip() + or reviewed_utc != reviewed_utc.strip() + ): + raise ValueError("invalid freeze review signoff") + try: + reviewed_time = datetime.fromisoformat(reviewed_utc.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("invalid freeze review timestamp") from error + if reviewed_time.tzinfo is None: + raise ValueError("freeze review timestamp lacks timezone") + if ( + type(lock.get("reports_collected_before_lock")) is not int + or lock["reports_collected_before_lock"] != 0 + ): + raise ValueError("freeze lock does not attest zero prior reports") + freeze_events = [event for event in event_records() if event["event"] == "freeze_locked"] + if len(freeze_events) > 1: + raise ValueError("multiple freeze-lock events") + if freeze_events: + freeze_sequence = freeze_events[0]["sequence"] + if any( + event["sequence"] < freeze_sequence and event["phase"] != "freeze" + for event in event_records() + ): + raise ValueError("evaluation activity predates the freeze-lock event") + elif any( + path.is_file() + for root in (COLLECTION, SCORING, RESULTS) + if root.exists() + for path in root.rglob("*") + ): + raise ValueError("evaluation artifacts exist before the freeze-lock event") + _AUTHENTICATED_FILE_DIGESTS = parse_file_manifest(FILE_MANIFEST.read_bytes()) + if "**Preregistration status:** FROZEN." not in read_frozen_text( + FREEZE / "plan.md" + ): + raise ValueError("plan is not marked FROZEN") + if announce: + print("static protocol validation passed") + + +def make_read_only(path: Path) -> None: + for item in sorted(path.rglob("*"), reverse=True): + if item.is_file(): + os.utime(item, (0, 0), follow_symlinks=False) + item.chmod(0o444) + elif item.is_dir(): + os.utime(item, (0, 0), follow_symlinks=False) + item.chmod(0o555) + os.utime(path, (0, 0), follow_symlinks=False) + path.chmod(0o555) + + +def resolve_run(run_id: str) -> tuple[dict[str, str], dict[str, str], dict[str, str]]: + schedule = load_schedule() + if run_id not in schedule: + raise ValueError(f"unknown run ID: {run_id}") + row = schedule[run_id] + target = load_target_map()[row["target_label"]] + condition = load_condition_map()[row["condition_label"]] + return row, target, condition + + +def event_records() -> list[dict[str, Any]]: + path = RUN / "events.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +def assert_authority_verification_allowed(wave: int) -> None: + if wave not in range(1, 6): + raise ValueError(f"invalid collection wave: {wave}") + events = event_records() + if any( + event["event"] == "authority_verified" + and event.get("details", {}).get("wave") == wave + for event in events + ): + raise ValueError(f"wave {wave} authority verification was already recorded") + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + completed = {event.get("run_id") for event in events if event["event"] == "report_preserved"} + required_completed = { + row["run_id"] for row in schedule_rows if int(row["wave"]) < wave + } + if not required_completed <= completed: + raise ValueError(f"earlier collection waves are incomplete before wave {wave}") + prepared = {event.get("run_id") for event in events if event["event"] == "cell_prepared"} + current = {row["run_id"] for row in schedule_rows if int(row["wave"]) == wave} + if prepared & current: + raise ValueError(f"wave {wave} authority verification must precede cell preparation") + + +def assert_prepare_allowed(run_id: str) -> None: + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + order = [row["run_id"] for row in schedule_rows] + position = order.index(run_id) + row = schedule_rows[position] + events = event_records() + authority_events = [ + event + for event in events + if event["event"] == "authority_verified" + and event.get("details", {}).get("wave") == int(row["wave"]) + and event.get("sha256") == sha256_file(FREEZE / "authority-manifest.tsv") + ] + if len(authority_events) != 1: + raise ValueError(f"wave {row['wave']} lacks one current authority verification") + prepared = {event.get("run_id") for event in events if event["event"] == "cell_prepared"} + completed = {event.get("run_id") for event in events if event["event"] == "report_preserved"} + if run_id in prepared: + raise ValueError(f"cell already prepared: {run_id}") + earlier_in_wave = { + prior["run_id"] + for prior in schedule_rows[:position] + if prior["wave"] == row["wave"] + } + if not earlier_in_wave <= prepared: + raise ValueError(f"prepare order violation before {run_id}") + earlier_waves = { + prior["run_id"] for prior in schedule_rows if int(prior["wave"]) < int(row["wave"]) + } + if not earlier_waves <= completed: + raise ValueError(f"wave barrier violation before {run_id}") + if len(prepared - completed) >= 3: + raise ValueError(f"three report cells are already active before {run_id}") + + +def record_agent_start(run_id: str, attempt: int, agent_id: str) -> None: + if attempt < 1: + raise ValueError("report attempt must be positive") + if not agent_id: + raise ValueError("agent ID must be nonempty") + events = event_records() + if not any(event["event"] == "cell_prepared" and event.get("run_id") == run_id for event in events): + raise ValueError(f"cell was not prepared: {run_id}") + key = (run_id, attempt) + started = { + (event.get("run_id"), event.get("attempt")) + for event in events + if event["event"] == "agent_started" + } + returned = { + (event.get("run_id"), event.get("attempt")) + for event in events + if event["event"] == "agent_returned" + } + if key in started: + raise ValueError(f"agent attempt already started: {run_id}/{attempt}") + used_agent_ids = { + event.get("agent_id") + for event in events + if event["event"] in {"agent_started", "evaluator_started"} + } + if agent_id in used_agent_ids: + raise ValueError(f"agent ID was already used by an evaluated agent: {agent_id}") + if len(started - returned) >= 3: + raise ValueError("three report agents are already active") + schedule_rows = read_frozen_tsv(SEALED / "launch-schedule.tsv") + position = next(index for index, row in enumerate(schedule_rows) if row["run_id"] == run_id) + current_wave = schedule_rows[position]["wave"] + earlier_in_wave = [ + row["run_id"] for row in schedule_rows[:position] if row["wave"] == current_wave + ] + started_runs = {item[0] for item in started} + if not all(prior in started_runs for prior in earlier_in_wave): + raise ValueError(f"agent launch-order violation before {run_id}") + if attempt > 1 and not any( + event["event"] == "infrastructure_failure" + and event.get("run_id") == run_id + and event.get("attempt") == attempt - 1 + for event in events + ): + raise ValueError("fresh attempt lacks a preceding infrastructure failure") + setup = json.loads((COLLECTION / "setups" / f"{run_id}.json").read_text()) + runtime = Path(setup["runtime_root"]) + verify_runtime(run_id, runtime) + prompt_digest = sha256_bytes(render_report_prompt(run_id, runtime).encode()) + if setup.get("report_prompt_sha256") != prompt_digest: + raise ValueError(f"rendered report prompt changed for {run_id}") + output_entries_now = list((runtime / "output").iterdir()) + if output_entries_now: + raise ValueError(f"report output is not initially empty: {run_id}/{attempt}") + append_event( + "collection", + "agent_started", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={ + "model": "gpt-5.6-sol", + "reasoning_effort": "ultra", + "fork_turns": "none", + "prompt_sha256": prompt_digest, + }, + ) + + +def record_prelaunch_failure(run_id: str, evidence: str) -> None: + if not evidence.strip(): + raise ValueError("prelaunch failure requires nonempty evidence") + resolve_run(run_id) + events = event_records() + if not any( + event["event"] == "cell_prepared" and event.get("run_id") == run_id + for event in events + ): + raise ValueError(f"cell was not prepared: {run_id}") + started = [ + event + for event in events + if event["event"] == "agent_started" and event.get("run_id") == run_id + ] + returned_attempts = { + event.get("attempt") + for event in events + if event["event"] == "agent_returned" and event.get("run_id") == run_id + } + if any(event["attempt"] not in returned_attempts for event in started): + raise ValueError("cannot record a prelaunch failure while a report agent is active") + if any( + event["event"] == "report_preserved" and event.get("run_id") == run_id + for event in events + ): + raise ValueError("cannot record a prelaunch failure after canonical completion") + next_attempt = max((event["attempt"] for event in started), default=0) + 1 + append_event( + "collection", + "prelaunch_failure", + run_id=run_id, + details={ + "disposition": "API_NO_AGENT_START", + "next_attempt": next_attempt, + "evidence": evidence, + }, + ) + + +def assert_agent_started(run_id: str, attempt: int, agent_id: str) -> None: + matches = [ + event + for event in event_records() + if event["event"] == "agent_started" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + and event.get("agent_id") == agent_id + ] + if len(matches) != 1: + raise ValueError(f"missing unique agent-start record: {run_id}/{attempt}/{agent_id}") + + +def record_reminder(run_id: str, attempt: int, agent_id: str) -> None: + assert_agent_started(run_id, attempt, agent_id) + events = event_records() + if any( + event["event"] == "reminder_sent" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + for event in events + ): + raise ValueError("the one permitted reminder was already recorded") + if any( + event["event"] == "agent_returned" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + for event in events + ): + raise ValueError("a reminder cannot follow agent return") + start = next( + event + for event in events + if event["event"] == "agent_started" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + ) + started = datetime.fromisoformat(start["time_utc"].replace("Z", "+00:00")) + if (datetime.now(timezone.utc) - started).total_seconds() < 180: + raise ValueError("reminder is not permitted before 180 seconds") + append_event( + "collection", + "reminder_sent", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"text_sha256": sha256_bytes(report_reminder_text().encode())}, + ) + + +def assert_evaluator_attempt_allowed( + kind: str, identity: str, attempt: int, events: list[dict[str, Any]] | None = None +) -> None: + if kind not in {"scorer", "adjudicator"} or attempt < 1: + raise ValueError(f"invalid evaluator attempt: {kind}/{identity}/{attempt}") + events = event_records() if events is None else events + if any( + event["event"] == "evaluator_started" + and event.get("attempt") == attempt + and event.get("details", {}).get("kind") == kind + and event.get("details", {}).get("identity") == identity + for event in events + ): + raise ValueError(f"evaluator attempt already started: {kind}/{identity}/{attempt}") + completion_event = "score_preserved" if kind == "scorer" else "adjudication_preserved" + if any( + event["event"] == completion_event + and ( + (kind == "scorer" and f"{event.get('details', {}).get('mode')}-{event.get('details', {}).get('scorer')}" == identity) + or (kind == "adjudicator" and event.get("details", {}).get("mode") == identity) + ) + for event in events + ): + raise ValueError(f"evaluator identity already completed: {kind}/{identity}") + if attempt > 1 and not any( + event["event"] == "evaluator_infrastructure_failure" + and event.get("attempt") == attempt - 1 + and event.get("details", {}).get("kind") == kind + and event.get("details", {}).get("identity") == identity + for event in events + ): + raise ValueError("fresh evaluator retry lacks a preceding infrastructure failure") + + +def record_evaluator_start(kind: str, identity: str, attempt: int, agent_id: str) -> None: + if kind not in {"scorer", "adjudicator"}: + raise ValueError(f"unknown evaluator kind: {kind}") + events = event_records() + assert_evaluator_attempt_allowed(kind, identity, attempt, events) + key = (kind, identity, attempt) + started = { + ( + event.get("details", {}).get("kind"), + event.get("details", {}).get("identity"), + event.get("attempt"), + ) + for event in events + if event["event"] == "evaluator_started" + } + returned = { + ( + event.get("details", {}).get("kind"), + event.get("details", {}).get("identity"), + event.get("attempt"), + ) + for event in events + if event["event"] == "evaluator_returned" + } + used_agent_ids = { + event.get("agent_id") + for event in events + if event["event"] in {"agent_started", "evaluator_started"} + } + if agent_id in used_agent_ids: + raise ValueError(f"agent ID was already used by an evaluated agent: {agent_id}") + if len(started - returned) >= 3: + raise ValueError("three evaluator agents are already active") + if kind == "scorer": + claims = [row["claim"] for row in read_frozen_tsv(SEALED / "scoring-schedule.tsv")] + if identity not in claims: + raise ValueError(f"unknown scorer claim: {identity}") + prior = claims[: claims.index(identity)] + started_identities = {(item[0], item[1]) for item in started} + if not all(("scorer", claim) in started_identities for claim in prior): + raise ValueError(f"scorer launch-order violation before {identity}") + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + verify_score_packet(mode, scorer) + else: + source_packet = SCORING / "adjudication-packets" / identity + if not source_packet.exists(): + raise ValueError(f"adjudication packet does not exist: {identity}") + verify_adjudication_packet(identity) + output = expected_evaluator_output(kind, identity, attempt) + verify_evaluator_runtime(kind, identity, attempt, source_packet, output) + if output.is_symlink() or not output.is_dir() or any(output.iterdir()): + raise ValueError(f"evaluator output is not initially empty: {kind}/{identity}") + append_event( + "scoring" if kind == "scorer" else "adjudication", + "evaluator_started", + attempt=attempt, + agent_id=agent_id, + details={ + "kind": kind, + "identity": identity, + "model": "gpt-5.6-sol", + "reasoning_effort": "ultra", + "fork_turns": "none", + "prompt_sha256": sha256_bytes( + render_packet_prompt( + "scorer.md" if kind == "scorer" else "adjudicator.md", + expected_evaluator_packet(kind, identity, attempt), + output, + **({"SCORER_ID": identity.split("-", 1)[1]} if kind == "scorer" else {}), + ).encode() + ), + }, + ) + + +def assert_evaluator_started( + kind: str, identity: str, attempt: int, agent_id: str +) -> None: + matches = [ + event + for event in event_records() + if event["event"] == "evaluator_started" + and event.get("attempt") == attempt + and event.get("agent_id") == agent_id + and event.get("details", {}).get("kind") == kind + and event.get("details", {}).get("identity") == identity + ] + if len(matches) != 1: + raise ValueError( + f"missing evaluator-start record: {kind}/{identity}/{attempt}/{agent_id}" + ) + + +def record_evaluator_prelaunch_failure( + kind: str, identity: str, attempt: int, output: Path, evidence: str +) -> None: + if not evidence.strip(): + raise ValueError("evaluator prelaunch failure requires nonempty evidence") + assert_evaluator_attempt_allowed(kind, identity, attempt) + if kind == "scorer": + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + verify_score_packet(mode, scorer) + else: + source_packet = SCORING / "adjudication-packets" / identity + verify_adjudication_packet(identity) + verify_evaluator_runtime(kind, identity, attempt, source_packet, output) + if any(output.iterdir()): + raise ValueError("prelaunch evaluator output is not empty") + append_event( + "scoring" if kind == "scorer" else "adjudication", + "evaluator_prelaunch_failure", + attempt=attempt, + details={ + "kind": kind, + "identity": identity, + "disposition": "API_NO_AGENT_START", + "evidence": evidence, + }, + ) + + +def prepare_cell(run_id: str, runtime: Path) -> None: + validate_static(require_lock=True, announce=False) + assert_prepare_allowed(run_id) + if runtime.exists(): + raise FileExistsError(f"runtime already exists: {runtime}") + row, target, condition = resolve_run(run_id) + expected_runtime = Path("/tmp/ur-eval") / row["cell_id"] + if runtime != expected_runtime: + raise ValueError(f"runtime must be the frozen neutral path {expected_runtime}") + package_source = EVALS / condition["package_path"] + target_source = EVALS / target["source_path"] + runtime.mkdir(parents=True) + shutil.copytree(package_source, runtime / "package") + shutil.copytree(target_source, runtime / "target") + shutil.copy2(FREEZE / "allowlists" / f"{target['mode']}.txt", runtime / "allowlist.txt") + (runtime / "output").mkdir() + if tar_tree_digest(runtime / "package") != condition["tree_sha256"]: + raise ValueError(f"package runtime copy does not match frozen identity for {run_id}") + if tar_tree_digest(runtime / "target") != target["tree_sha256"]: + raise ValueError(f"target runtime copy does not match frozen identity for {run_id}") + package_bytes = byte_tree_digest(runtime / "package") + target_bytes = byte_tree_digest(runtime / "target") + if package_bytes != byte_tree_digest(package_source): + raise ValueError(f"package runtime copy differs for {run_id}") + if target_bytes != byte_tree_digest(target_source): + raise ValueError(f"target runtime copy differs for {run_id}") + expected_allowlist = frozen_file_digest( + FREEZE / "allowlists" / f"{target['mode']}.txt" + ) + if sha256_file(runtime / "allowlist.txt") != expected_allowlist: + raise ValueError(f"allowlist runtime copy differs from frozen bytes for {run_id}") + attestation = { + "schema_version": 1, + "run_id": run_id, + "cell_id": row["cell_id"], + "runtime_root": str(runtime), + "package_byte_tree_sha256": package_bytes, + "target_byte_tree_sha256": target_bytes, + "allowlist_sha256": expected_allowlist, + "report_prompt_sha256": sha256_bytes(render_report_prompt(run_id, runtime).encode()), + "output_initially_empty": True, + "prepared_utc": utc_now(), + } + setup_path = COLLECTION / "setups" / f"{run_id}.json" + write_once(setup_path, json_dump(attestation)) + setup_path.chmod(0o444) + make_read_only(runtime / "package") + make_read_only(runtime / "target") + os.utime(runtime / "allowlist.txt", (0, 0), follow_symlinks=False) + (runtime / "allowlist.txt").chmod(0o444) + os.utime(runtime, (0, 0), follow_symlinks=False) + runtime.chmod(0o555) + append_event( + "collection", + "cell_prepared", + run_id=run_id, + digest=sha256_file(setup_path), + details={"cell_id": row["cell_id"]}, + ) + print(runtime) + + +def report_prompt_blocks() -> list[str]: + template = read_frozen_text(FREEZE / "prompts" / "report.md") + blocks = re.findall(r"```text\n(.*?)\n```", template, flags=re.DOTALL) + if len(blocks) != 2: + raise ValueError("report prompt template must contain prompt and reminder fences") + return blocks + + +def report_reminder_text() -> str: + return report_prompt_blocks()[1] + + +def render_report_prompt(run_id: str, runtime: Path) -> str: + _row, target, _condition = resolve_run(run_id) + prompt = report_prompt_blocks()[0] + replacements = { + "[PACKAGE]": str(runtime / "package"), + "[TARGET]": str(runtime / "target"), + "[URL_ALLOWLIST]": str(runtime / "allowlist.txt"), + "[OUTPUT]": str(runtime / "output"), + "[WORD_LIMIT]": target["word_cap"], + } + for old, new in replacements.items(): + prompt = prompt.replace(old, new) + if re.search(r"\[[A-Z_]+\]", prompt): + raise ValueError("unresolved report-prompt placeholder") + return prompt + + +def verify_runtime(run_id: str, runtime: Path, *, allow_invalid_output: bool = False) -> None: + setup_path = COLLECTION / "setups" / f"{run_id}.json" + prepared = [ + event + for event in event_records() + if event["event"] == "cell_prepared" and event.get("run_id") == run_id + ] + if len(prepared) != 1 or prepared[0].get("sha256") != sha256_file(setup_path): + raise ValueError(f"setup attestation changed for {run_id}") + setup = json.loads(setup_path.read_text()) + row, _target, _condition = resolve_run(run_id) + expected_runtime = Path("/tmp/ur-eval") / row["cell_id"] + if runtime != expected_runtime or runtime != Path(setup["runtime_root"]): + raise ValueError(f"runtime is not bound to frozen cell {run_id}") + if runtime.is_symlink() or not runtime.is_dir(): + raise ValueError(f"runtime root is not a real directory: {runtime}") + if runtime.stat().st_mode & 0o222: + raise ValueError(f"runtime root is writable: {runtime}") + required_inputs = {"package", "target", "allowlist.txt"} + expected_entries = required_inputs | {"output"} + actual_entries = {entry.name for entry in runtime.iterdir()} + if ( + (not allow_invalid_output and actual_entries != expected_entries) + or (allow_invalid_output and (not required_inputs <= actual_entries or actual_entries - expected_entries)) + ): + raise ValueError(f"runtime root inventory changed for {run_id}") + allowlist = runtime / "allowlist.txt" + output = runtime / "output" + if allowlist.is_symlink() or not stat.S_ISREG(allowlist.lstat().st_mode): + raise ValueError(f"runtime allowlist is not a real file: {run_id}") + if not allow_invalid_output and ( + not output.exists() + or output.is_symlink() + or not stat.S_ISDIR(output.lstat().st_mode) + ): + raise ValueError(f"runtime output is not a real directory: {run_id}") + checks = { + "package_byte_tree_sha256": byte_tree_digest(runtime / "package"), + "target_byte_tree_sha256": byte_tree_digest(runtime / "target"), + "allowlist_sha256": sha256_file(allowlist), + "report_prompt_sha256": sha256_bytes(render_report_prompt(run_id, runtime).encode()), + } + for field, actual in checks.items(): + if setup[field] != actual: + raise ValueError(f"runtime input changed for {run_id}: {field}") + + +def observe_path(path: Path) -> dict[str, Any]: + observation: dict[str, Any] = {"path": str(path)} + try: + mode = path.lstat().st_mode + except FileNotFoundError: + observation["type"] = "missing" + return observation + observation["mode"] = stat.S_IMODE(mode) + if stat.S_ISLNK(mode): + observation.update({"type": "symlink", "target": os.readlink(path)}) + elif stat.S_ISREG(mode): + data = path.read_bytes() + observation.update( + {"type": "file", "bytes": len(data), "sha256": sha256_bytes(data)} + ) + elif stat.S_ISDIR(mode): + observation["type"] = "directory" + observation["entries"] = sorted(entry.name for entry in path.iterdir()) + try: + observation["byte_tree_sha256"] = byte_tree_digest(path) + except (OSError, ValueError) as error: + observation["byte_tree_error"] = f"{type(error).__name__}: {error}" + else: + try: + observation["tar_tree_sha256"] = tar_tree_digest(path) + except (OSError, subprocess.SubprocessError, ValueError) as error: + observation["tar_tree_error"] = f"{type(error).__name__}: {error}" + else: + observation.update({"type": "special", "file_type": stat.S_IFMT(mode)}) + return observation + + +def report_runtime_forensics(run_id: str, runtime: Path, error: Exception) -> dict[str, Any]: + row, target, condition = resolve_run(run_id) + expected_runtime = Path("/tmp/ur-eval") / row["cell_id"] + if runtime != expected_runtime: + raise ValueError(f"forensic runtime is not the frozen neutral path {expected_runtime}") + setup_path = COLLECTION / "setups" / f"{run_id}.json" + prepared = [ + event + for event in event_records() + if event["event"] == "cell_prepared" and event.get("run_id") == run_id + ] + return { + "schema_version": 1, + "run_id": run_id, + "verification_error": f"{type(error).__name__}: {error}", + "expected": { + "runtime_root": str(expected_runtime), + "setup_sha256": prepared[0].get("sha256") if len(prepared) == 1 else None, + "package_tar_tree_sha256": condition["tree_sha256"], + "target_tar_tree_sha256": target["tree_sha256"], + "allowlist_sha256": frozen_file_digest( + FREEZE / "allowlists" / f"{target['mode']}.txt" + ), + }, + "observed": { + "runtime_root": observe_path(runtime), + "setup": observe_path(setup_path), + "package": observe_path(runtime / "package") + if runtime.exists() and not runtime.is_symlink() and runtime.is_dir() + else {"type": "unavailable"}, + "target": observe_path(runtime / "target") + if runtime.exists() and not runtime.is_symlink() and runtime.is_dir() + else {"type": "unavailable"}, + "allowlist": observe_path(runtime / "allowlist.txt") + if runtime.exists() and not runtime.is_symlink() and runtime.is_dir() + else {"type": "unavailable"}, + }, + } + + +def append_event( + phase: str, + event: str, + *, + run_id: str | None = None, + attempt: int | None = None, + agent_id: str | None = None, + digest: str | None = None, + details: dict[str, Any] | None = None, +) -> None: + path = RUN / "events.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+", encoding="utf-8", newline="") as file: + fcntl.flock(file, fcntl.LOCK_EX) + file.seek(0) + lines = file.read().splitlines() + sequence = len(lines) + 1 + record: dict[str, Any] = { + "schema_version": 1, + "sequence": sequence, + "previous_event_sha256": sha256_bytes(lines[-1].encode()) if lines else None, + "time_utc": utc_now(), + "phase": phase, + "event": event, + "details": details or {}, + } + if run_id is not None: + record["run_id"] = run_id + if attempt is not None: + record["attempt"] = attempt + if agent_id is not None: + record["agent_id"] = agent_id + if digest is not None: + record["sha256"] = digest + validate_event_record(record, sequence) + file.seek(0, os.SEEK_END) + file.write(json.dumps(record, sort_keys=True) + "\n") + file.flush() + os.fsync(file.fileno()) + fcntl.flock(file, fcntl.LOCK_UN) + + +def validate_event_ledger() -> None: + path = RUN / "events.jsonl" + if not path.exists(): + return + previous: str | None = None + for index, line in enumerate(path.read_text().splitlines(), start=1): + value = json.loads(line) + validate_event_record(value, index) + if value.get("previous_event_sha256") != previous: + raise ValueError(f"broken event hash chain at line {index}") + previous = sha256_bytes(line.encode()) + + +def validate_event_record(value: Any, expected_sequence: int) -> None: + required = { + "schema_version", + "sequence", + "previous_event_sha256", + "time_utc", + "phase", + "event", + "details", + } + optional = {"run_id", "attempt", "agent_id", "sha256"} + if not isinstance(value, dict) or not required <= set(value) or not set(value) <= required | optional: + raise ValueError(f"invalid event fields at sequence {expected_sequence}") + if ( + type(value["schema_version"]) is not int + or value["schema_version"] != 1 + or type(value["sequence"]) is not int + or value["sequence"] != expected_sequence + ): + raise ValueError(f"invalid event sequence {expected_sequence}") + previous = value["previous_event_sha256"] + if previous is not None and ( + not isinstance(previous, str) or not re.fullmatch(r"[0-9a-f]{64}", previous) + ): + raise ValueError(f"invalid prior-event hash at sequence {expected_sequence}") + if not isinstance(value["time_utc"], str): + raise ValueError(f"invalid event timestamp at sequence {expected_sequence}") + try: + parsed_time = datetime.fromisoformat(value["time_utc"].replace("Z", "+00:00")) + except (AttributeError, ValueError) as error: + raise ValueError(f"invalid event timestamp at sequence {expected_sequence}") from error + if parsed_time.tzinfo is None: + raise ValueError(f"event timestamp lacks timezone at sequence {expected_sequence}") + if ( + not isinstance(value["phase"], str) + or value["phase"] not in EVENT_PHASES + or not isinstance(value["event"], str) + or not value["event"] + ): + raise ValueError(f"invalid event identity at sequence {expected_sequence}") + if not isinstance(value["details"], dict): + raise ValueError(f"invalid event details at sequence {expected_sequence}") + if "run_id" in value and (not isinstance(value["run_id"], str) or not value["run_id"]): + raise ValueError(f"invalid event run ID at sequence {expected_sequence}") + if "attempt" in value and ( + not isinstance(value["attempt"], int) + or isinstance(value["attempt"], bool) + or value["attempt"] < 1 + ): + raise ValueError(f"invalid event attempt at sequence {expected_sequence}") + if "agent_id" in value and ( + not is_nonblank_string(value["agent_id"]) + or value["agent_id"] != value["agent_id"].strip() + ): + raise ValueError(f"invalid event agent ID at sequence {expected_sequence}") + if "sha256" in value and ( + not isinstance(value["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) + ): + raise ValueError(f"invalid event digest at sequence {expected_sequence}") + + +def validate_preserved_artifacts() -> None: + events = event_records() + attempt_events: dict[tuple[str, int], dict[str, Any]] = {} + evaluator_attempt_events: dict[tuple[str, str, int], dict[str, Any]] = {} + invalid_output_events: dict[tuple[str, str, int], dict[str, Any]] = {} + for event in events: + if event["event"] == "attempt_preserved": + key = (event["run_id"], event["attempt"]) + if key in attempt_events: + raise ValueError(f"duplicate attempt-preservation event: {key}") + attempt_events[key] = event + directory = ( + COLLECTION + / "attempts" + / event["run_id"] + / str(event["attempt"]) + ) + if byte_tree_digest(directory) != event.get("sha256"): + raise ValueError( + f"preserved attempt changed: {event['run_id']}/{event['attempt']}" + ) + elif event["event"] == "invalid_output_preserved": + details = event.get("details", {}) + kind = details.get("kind") + if kind not in {"scorer", "adjudicator"}: + raise ValueError("invalid preserved-output evaluator kind") + phase = "scoring" if kind == "scorer" else "adjudication" + identity = details.get("identity") + key = (phase, str(identity), event["attempt"]) + if event["phase"] != phase or key in invalid_output_events: + raise ValueError(f"duplicate or misphased invalid-output event: {key}") + invalid_output_events[key] = event + directory = ( + SCORING + / "invalid" + / phase + / str(identity) + / str(event.get("attempt")) + ) + if byte_tree_digest(directory) != event.get("sha256"): + raise ValueError(f"preserved invalid output changed: {directory}") + elif event["event"] == "evaluator_attempt_preserved": + details = event.get("details", {}) + kind = details.get("kind") + if kind not in {"scorer", "adjudicator"}: + raise ValueError("invalid preserved-attempt evaluator kind") + phase = "scoring" if kind == "scorer" else "adjudication" + identity = details.get("identity") + key = (phase, str(identity), event["attempt"]) + if event["phase"] != phase or key in evaluator_attempt_events: + raise ValueError(f"duplicate or misphased evaluator-attempt event: {key}") + evaluator_attempt_events[key] = event + directory = ( + SCORING + / "evaluator-attempts" + / phase + / str(identity) + / str(event.get("attempt")) + ) + if byte_tree_digest(directory) != event.get("sha256"): + raise ValueError(f"preserved evaluator attempt changed: {directory}") + elif event["event"] == "freeze_locked": + if ( + sha256_file(FILE_MANIFEST) != event.get("sha256") + or sha256_file(LOCK) != event.get("details", {}).get("lock_sha256") + ): + raise ValueError("freeze-lock artifacts changed") + elif event["event"] == "authority_verified": + if frozen_file_digest(FREEZE / "authority-manifest.tsv") != event.get("sha256"): + raise ValueError("authority verification event has the wrong manifest") + elif event["event"] == "report_preserved": + report = ( + COLLECTION + / "attempts" + / event["run_id"] + / str(event["attempt"]) + / "report.md" + ) + if sha256_file(report) != event.get("sha256"): + raise ValueError(f"preserved report changed: {event['run_id']}") + elif event["event"] == "collection_locked": + if sha256_file(COLLECTION / "valid-index.jsonl") != event.get("sha256"): + raise ValueError("locked collection index changed") + elif event["event"] == "blind_packet_preserved": + details = event.get("details", {}) + packet = SCORING / "packets" / str(details.get("mode")) / str( + details.get("scorer") + ) + if byte_tree_digest(packet) != event.get("sha256"): + raise ValueError(f"preserved blind packet changed: {packet}") + elif event["event"] == "score_preserved": + details = event.get("details", {}) + score = SCORING / "raw" / str(details.get("mode")) / ( + str(details.get("scorer")) + ".json" + ) + if sha256_file(score) != event.get("sha256"): + raise ValueError(f"preserved score changed: {score}") + elif event["event"] == "disagreements_materialized": + path = SCORING / "disagreements" / ( + str(event.get("details", {}).get("mode")) + ".json" + ) + if sha256_file(path) != event.get("sha256"): + raise ValueError(f"preserved disagreements changed: {path}") + elif event["event"] == "adjudication_packet_preserved": + packet = SCORING / "adjudication-packets" / str( + event.get("details", {}).get("mode") + ) + if byte_tree_digest(packet) != event.get("sha256"): + raise ValueError(f"preserved adjudication packet changed: {packet}") + elif event["event"] == "adjudication_preserved": + path = SCORING / "adjudications" / ( + str(event.get("details", {}).get("mode")) + ".json" + ) + if sha256_file(path) != event.get("sha256"): + raise ValueError(f"preserved adjudication changed: {path}") + elif event["event"] == "final_blind_score_locked": + path = SCORING / "final" / ( + str(event.get("details", {}).get("mode")) + ".json" + ) + if sha256_file(path) != event.get("sha256"): + raise ValueError(f"final blind score changed: {path}") + elif event["event"] == "conditions_revealed": + if sha256_file(RUN / "unblinding.json") != event.get("sha256"): + raise ValueError("unblinding artifact changed") + elif event["event"] == "aggregate_written": + if ( + sha256_file(RESULTS / "aggregate.json") != event.get("sha256") + or sha256_file(RESULTS / "summary.md") + != event.get("details", {}).get("summary_sha256") + ): + raise ValueError("final result artifacts changed") + elif event["event"] == "run_invalidated": + if sha256_file(RUN / "INVALID.json") != event.get("sha256"): + raise ValueError("INVALID marker changed") + attempt_root = COLLECTION / "attempts" + actual_attempts: set[tuple[str, int]] = set() + if attempt_root.exists(): + for run_dir in attempt_root.iterdir(): + if run_dir.is_symlink() or not run_dir.is_dir(): + raise ValueError(f"invalid attempt run directory: {run_dir}") + for attempt_dir in run_dir.iterdir(): + if ( + attempt_dir.is_symlink() + or not attempt_dir.is_dir() + or not attempt_dir.name.isdigit() + or int(attempt_dir.name) < 1 + ): + raise ValueError(f"invalid attempt directory: {attempt_dir}") + actual_attempts.add((run_dir.name, int(attempt_dir.name))) + if actual_attempts != set(attempt_events): + raise ValueError("attempt directories and preservation events differ") + + def inventory_evaluator_attempts( + root: Path, expected: set[tuple[str, str, int]], label: str + ) -> set[tuple[str, str, int]]: + if root.is_symlink(): + raise ValueError(f"invalid {label} artifact root: {root}") + if not root.exists(): + if expected: + raise ValueError(f"missing {label} artifact root") + return set() + if not root.is_dir(): + raise ValueError(f"invalid {label} artifact root: {root}") + if not expected: + raise ValueError(f"orphan empty {label} artifact root: {root}") + actual: set[tuple[str, str, int]] = set() + actual_phases: set[str] = set() + actual_identities: set[tuple[str, str]] = set() + scorer_identities = { + f"{mode}-{scorer}" for mode in MODES for scorer in SCORERS + } + for phase_dir in root.iterdir(): + if ( + phase_dir.is_symlink() + or not phase_dir.is_dir() + or phase_dir.name not in {"scoring", "adjudication"} + ): + raise ValueError(f"invalid {label} phase directory: {phase_dir}") + phase = phase_dir.name + actual_phases.add(phase) + for identity_dir in phase_dir.iterdir(): + identity = identity_dir.name + valid_identity = ( + identity in scorer_identities if phase == "scoring" else identity in MODES + ) + if identity_dir.is_symlink() or not identity_dir.is_dir() or not valid_identity: + raise ValueError(f"invalid {label} identity directory: {identity_dir}") + actual_identities.add((phase, identity)) + for attempt_dir in identity_dir.iterdir(): + if ( + attempt_dir.is_symlink() + or not attempt_dir.is_dir() + or not re.fullmatch(r"[1-9][0-9]*", attempt_dir.name) + ): + raise ValueError(f"invalid {label} attempt directory: {attempt_dir}") + actual.add((phase, identity, int(attempt_dir.name))) + expected_phases = {phase for phase, _identity, _attempt in expected} + expected_identities = {(phase, identity) for phase, identity, _attempt in expected} + if actual_phases != expected_phases or actual_identities != expected_identities: + raise ValueError(f"{label} directory hierarchy differs from preservation events") + return actual + + actual_evaluator_attempts = inventory_evaluator_attempts( + SCORING / "evaluator-attempts", + set(evaluator_attempt_events), + "evaluator-attempt", + ) + if actual_evaluator_attempts != set(evaluator_attempt_events): + raise ValueError("evaluator-attempt directories and preservation events differ") + actual_invalid_outputs = inventory_evaluator_attempts( + SCORING / "invalid", set(invalid_output_events), "invalid-output" + ) + if actual_invalid_outputs != set(invalid_output_events): + raise ValueError("invalid-output directories and preservation events differ") + if set(evaluator_attempt_events) & set(invalid_output_events): + raise ValueError("one evaluator attempt has both valid/infra and invalid preservation") + + invalidated = [event for event in events if event["event"] == "run_invalidated"] + invalid_returns = [ + event + for event in events + if event["event"] == "evaluator_returned" + and event.get("details", {}).get("api_state") == "INVALID_OUTPUT" + ] + invalid_marker = RUN / "INVALID.json" + if not invalid_output_events: + if invalid_marker.exists() or invalid_marker.is_symlink() or invalidated or invalid_returns: + raise ValueError("INVALID marker or terminal-invalid events lack an invalid output") + else: + if len(invalid_output_events) != 1 or len(invalidated) != 1 or len(invalid_returns) != 1: + raise ValueError("terminal-invalid evaluator state is not unique and complete") + if invalid_marker.is_symlink() or not invalid_marker.is_file(): + raise ValueError("terminal-invalid event lacks a real INVALID marker") + key, invalid_event = next(iter(invalid_output_events.items())) + phase, identity, attempt = key + invalid_directory = SCORING / "invalid" / phase / identity / str(attempt) + attestation = json.loads((invalid_directory / "attestation.json").read_text()) + marker = json.loads(invalid_marker.read_text()) + if marker != attestation: + raise ValueError("INVALID marker differs from the invalid-output attestation") + if ( + marker.get("phase") != phase + or marker.get("identity") != identity + or marker.get("attempt") != attempt + or marker.get("agent_id") != invalid_event.get("agent_id") + or marker.get("disposition") != "INVALID_NONRERUNNABLE_EVALUATOR_OUTPUT" + ): + raise ValueError("terminal-invalid attestation identity is inconsistent") + invalidated_event = invalidated[0] + if ( + invalidated_event.get("phase") != phase + or invalidated_event.get("attempt") != attempt + or invalidated_event.get("agent_id") != marker.get("agent_id") + or invalidated_event.get("sha256") != sha256_file(invalid_marker) + or invalidated_event.get("details") + != {"identity": identity, "evidence": marker.get("evidence")} + ): + raise ValueError("run-invalidated event differs from the INVALID marker") + returned = invalid_returns[0] + expected_kind = "scorer" if phase == "scoring" else "adjudicator" + if ( + returned.get("phase") != phase + or returned.get("attempt") != attempt + or returned.get("agent_id") != marker.get("agent_id") + or returned.get("details") + != {"kind": expected_kind, "identity": identity, "api_state": "INVALID_OUTPUT"} + ): + raise ValueError("invalid evaluator-return event is inconsistent") + + setup_root = COLLECTION / "setups" + actual_setups: set[str] = set() + if setup_root.exists(): + for path in setup_root.iterdir(): + if path.is_symlink() or not path.is_file() or path.suffix != ".json": + raise ValueError(f"invalid setup artifact: {path}") + actual_setups.add(path.stem) + prepared = { + event["run_id"]: event + for event in events + if event["event"] == "cell_prepared" + } + if len(prepared) != sum(event["event"] == "cell_prepared" for event in events): + raise ValueError("duplicate cell-preparation event") + if actual_setups != set(prepared): + raise ValueError("cell setup files and preparation events differ") + for run_id, event in prepared.items(): + if sha256_file(setup_root / f"{run_id}.json") != event.get("sha256"): + raise ValueError(f"cell setup changed: {run_id}") + + +def append_report_index(metadata: dict[str, Any]) -> None: + path = COLLECTION / "valid-index.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+", encoding="utf-8", newline="") as file: + fcntl.flock(file, fcntl.LOCK_EX) + file.seek(0) + existing = [json.loads(line) for line in file.read().splitlines()] + if any(item["run_id"] == metadata["run_id"] for item in existing): + raise ValueError(f"canonical report already indexed for {metadata['run_id']}") + file.seek(0, os.SEEK_END) + file.write(json.dumps(metadata, sort_keys=True) + "\n") + file.flush() + os.fsync(file.fileno()) + fcntl.flock(file, fcntl.LOCK_UN) + + +def output_entries(output: Path) -> list[Path]: + if not output.exists() or output.is_symlink() or not stat.S_ISDIR(output.lstat().st_mode): + raise ValueError(f"output path is not a real directory: {output}") + entries = sorted(output.iterdir(), key=lambda path: path.name) if output.exists() else [] + for entry in entries: + mode = entry.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISREG(mode): + raise ValueError(f"unsupported output entry: {entry}") + return entries + + +def snapshot_path(source: Path, raw_root: Path) -> list[dict[str, Any]]: + """Preserve every regular byte reachable without following a symlink.""" + raw_root.mkdir(parents=True, exist_ok=True) + records: list[dict[str, Any]] = [] + try: + root_mode = source.lstat().st_mode + except FileNotFoundError: + return [{"path": ".", "type": "missing"}] + if stat.S_ISLNK(root_mode): + return [{"path": ".", "type": "symlink", "target": os.readlink(source)}] + if stat.S_ISREG(root_mode): + data = source.read_bytes() + stored = "__output_path_file__" + write_bytes_once(raw_root / stored, data) + return [ + { + "path": ".", + "type": "file", + "stored_as": stored, + "bytes": len(data), + "sha256": sha256_bytes(data), + } + ] + if not stat.S_ISDIR(root_mode): + return [{"path": ".", "type": "special", "mode": stat.S_IFMT(root_mode)}] + + records.append({"path": ".", "type": "directory"}) + + def walk(directory: Path, relative: Path) -> None: + with os.scandir(directory) as iterator: + entries = sorted(iterator, key=lambda entry: entry.name) + for entry in entries: + child_relative = relative / entry.name + rendered = child_relative.as_posix() + mode = entry.stat(follow_symlinks=False).st_mode + source_child = directory / entry.name + destination_child = raw_root / child_relative + if stat.S_ISREG(mode): + data = source_child.read_bytes() + write_bytes_once(destination_child, data) + records.append( + { + "path": rendered, + "type": "file", + "bytes": len(data), + "sha256": sha256_bytes(data), + } + ) + elif stat.S_ISDIR(mode): + destination_child.mkdir() + records.append({"path": rendered, "type": "directory"}) + walk(source_child, child_relative) + elif stat.S_ISLNK(mode): + records.append( + {"path": rendered, "type": "symlink", "target": os.readlink(source_child)} + ) + else: + records.append( + {"path": rendered, "type": "special", "mode": stat.S_IFMT(mode)} + ) + + walk(source, Path()) + return records + + +def preserve_captured_file(raw_root: Path, filename: str, data: bytes) -> list[dict[str, Any]]: + """Preserve already-captured bytes as an exact one-file output snapshot.""" + if Path(filename).name != filename or filename in {"", ".", ".."}: + raise ValueError(f"invalid captured output filename: {filename}") + raw_root.mkdir(parents=True, exist_ok=False) + write_bytes_once(raw_root / filename, data) + return [ + {"path": ".", "type": "directory"}, + { + "path": filename, + "type": "file", + "bytes": len(data), + "sha256": sha256_bytes(data), + }, + ] + + +def record_report( + run_id: str, + attempt: int, + runtime: Path, + agent_id: str, + scope_deviation: bool, + scope_evidence: str, +) -> None: + if scope_deviation and not scope_evidence.strip(): + raise ValueError("scope deviation requires nonempty evidence") + assert_agent_started(run_id, attempt, agent_id) + verify_runtime(run_id, runtime) + output = runtime / "output" + if not output.exists() or output.is_symlink() or not stat.S_ISDIR(output.lstat().st_mode): + raise ValueError(f"output path is not a real directory: {output}") + entries = output_entries(output) + if [entry.name for entry in entries] != ["report.md"]: + raise ValueError(f"{run_id} output is not exactly one report.md") + data = entries[0].read_bytes() + text = data.decode("utf-8") + if not text.strip(): + raise ValueError(f"{run_id} report.md is empty or whitespace-only") + words = len(text.split()) + _row, target, _condition = resolve_run(run_id) + destination = COLLECTION / "attempts" / run_id / str(attempt) + if destination.exists(): + raise FileExistsError(f"attempt already preserved: {destination}") + destination.mkdir(parents=True) + write_bytes_once(destination / "report.md", data) + metadata = { + "schema_version": 1, + "run_id": run_id, + "attempt": attempt, + "agent_id": agent_id, + "report_sha256": sha256_bytes(data), + "word_count": words, + "word_cap": int(target["word_cap"]), + "within_word_cap": words <= int(target["word_cap"]), + "utf8": True, + "canonical_for_scoring": True, + "semantic_noncompletion": False, + "terminal_disposition": "COMPLETE", + "api_state": "COMPLETED", + "operational_scope_deviation": scope_deviation, + "scope_evidence": scope_evidence, + "source_isolation": "procedural", + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(metadata)) + append_report_index(metadata) + make_read_only(destination) + append_event( + "collection", + "attempt_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"disposition": "COMPLETE"}, + ) + append_event( + "collection", + "agent_returned", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"api_state": "COMPLETED"}, + ) + append_event( + "collection", + "report_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=metadata["report_sha256"], + details={ + "word_count": words, + "within_word_cap": metadata["within_word_cap"], + "operational_scope_deviation": scope_deviation, + "disposition": "COMPLETE", + }, + ) + print(json_dump(metadata), end="") + + +def preserve_failed_report_attempt( + run_id: str, + attempt: int, + runtime: Path, + agent_id: str, + disposition: str, + evidence: str, + scope_deviation: bool, +) -> None: + if disposition not in INFRA_FAILURE_CODES | TERMINAL_REPORT_FAILURE_CODES: + raise ValueError(f"unknown failure disposition: {disposition}") + if not evidence.strip(): + raise ValueError("failed report disposition requires nonempty evidence") + assert_agent_started(run_id, attempt, agent_id) + forensics: dict[str, Any] | None = None + try: + verify_runtime(run_id, runtime, allow_invalid_output=True) + except (OSError, ValueError) as error: + forensics = report_runtime_forensics(run_id, runtime, error) + disposition = "INVALID_OUTPUT" + scope_deviation = True + evidence = ( + f"{evidence} Runtime/input verification failed: " + f"{forensics['verification_error']}" + ) + output = runtime / "output" + destination = COLLECTION / "attempts" / run_id / str(attempt) + if destination.exists(): + raise FileExistsError(f"attempt already preserved: {destination}") + if forensics is None: + entry_manifest = snapshot_path(output, destination / "raw-output") + manifest_path = destination / "raw-output-manifest.json" + captured_report = destination / "raw-output" / "report.md" + else: + # A failed runtime check may have been caused by an unexpected root entry + # or by drift in an input tree. Preserve the entire neutral runtime without + # following symlinks, rather than retaining only the ordinary output path. + entry_manifest = snapshot_path(runtime, destination / "raw-runtime") + manifest_path = destination / "raw-runtime-manifest.json" + setup = COLLECTION / "setups" / f"{run_id}.json" + setup_manifest = snapshot_path(setup, destination / "setup-at-verification") + write_once( + destination / "setup-at-verification-manifest.json", + json_dump(setup_manifest), + ) + captured_report = destination / "raw-runtime" / "output" / "report.md" + usable_report: bytes | None = None + if ( + captured_report.exists() + and not captured_report.is_symlink() + and stat.S_ISREG(captured_report.lstat().st_mode) + ): + data = captured_report.read_bytes() + try: + data.decode("utf-8") + except UnicodeDecodeError: + pass + else: + usable_report = data + write_once(manifest_path, json_dump(entry_manifest)) + if forensics is not None: + write_once(destination / "runtime-forensics.json", json_dump(forensics)) + infrastructure = disposition in INFRA_FAILURE_CODES + if infrastructure: + metadata = { + "schema_version": 1, + "run_id": run_id, + "attempt": attempt, + "agent_id": agent_id, + "terminal_disposition": disposition, + "api_state": "INFRASTRUCTURE_FAILURE", + "rerunnable": True, + "evidence": evidence, + "operational_scope_deviation": scope_deviation, + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(metadata)) + make_read_only(destination) + append_event( + "collection", + "attempt_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"disposition": disposition}, + ) + append_event( + "collection", + "agent_returned", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"api_state": "INFRASTRUCTURE_FAILURE"}, + ) + append_event( + "collection", + "infrastructure_failure", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"disposition": disposition, "evidence": evidence}, + ) + runtime.chmod(0o755) + try: + if output.is_symlink() or output.is_file(): + output.unlink() + elif output.exists(): + shutil.rmtree(output) + output.mkdir() + finally: + os.utime(runtime, (0, 0), follow_symlinks=False) + runtime.chmod(0o555) + print(json_dump(metadata), end="") + return + + if usable_report is None: + usable_report = ( + "# Evaluator-marked failed replicate\n\n" + f"No usable canonical report was produced. Terminal disposition: {disposition}.\n" + ).encode() + text = usable_report.decode("utf-8") + words = len(text.split()) + _row, target, _condition = resolve_run(run_id) + write_bytes_once(destination / "report.md", usable_report) + metadata = { + "schema_version": 1, + "run_id": run_id, + "attempt": attempt, + "agent_id": agent_id, + "report_sha256": sha256_bytes(usable_report), + "word_count": words, + "word_cap": int(target["word_cap"]), + "within_word_cap": words <= int(target["word_cap"]), + "utf8": True, + "canonical_for_scoring": True, + "semantic_noncompletion": True, + "terminal_disposition": disposition, + "api_state": "TERMINAL_NONCOMPLETION", + "operational_scope_deviation": scope_deviation or disposition == "INVALID_OUTPUT", + "scope_evidence": evidence, + "source_isolation": "procedural", + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(metadata)) + append_report_index(metadata) + make_read_only(destination) + append_event( + "collection", + "attempt_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"disposition": disposition}, + ) + append_event( + "collection", + "agent_returned", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + details={"api_state": "TERMINAL_NONCOMPLETION"}, + ) + append_event( + "collection", + "report_preserved", + run_id=run_id, + attempt=attempt, + agent_id=agent_id, + digest=metadata["report_sha256"], + details={"disposition": disposition, "semantic_noncompletion": True}, + ) + print(json_dump(metadata), end="") + + +def verify_collection_lock() -> None: + path = COLLECTION / "valid-index.jsonl" + matches = [event for event in event_records() if event["event"] == "collection_locked"] + if len(matches) != 1 or matches[0].get("sha256") != sha256_file(path): + raise ValueError("canonical collection index is not locked to the event ledger") + if matches[0].get("details", {}).get("report_count") != 80: + raise ValueError("collection lock has an invalid report count") + + +def load_index(*, require_collection_lock: bool = False) -> dict[str, dict[str, Any]]: + path = COLLECTION / "valid-index.jsonl" + if path.is_symlink() or not path.is_file(): + raise ValueError("canonical report index is not a real file") + rows = [json.loads(line) for line in path.read_text().splitlines()] + if any(not isinstance(row, dict) or not isinstance(row.get("run_id"), str) for row in rows): + raise ValueError("canonical report index has an invalid row") + result = {row["run_id"]: row for row in rows} + if len(rows) != len(result): + raise ValueError("duplicate canonical reports in index") + if require_collection_lock: + verify_collection_lock() + for run_id, row in result.items(): + if ( + not isinstance(row, dict) + or type(row.get("schema_version")) is not int + or row["schema_version"] != 1 + or type(row.get("attempt")) is not int + or row["attempt"] < 1 + or not isinstance(row.get("agent_id"), str) + or not row["agent_id"] + or not isinstance(row.get("report_sha256"), str) + or not re.fullmatch(r"[0-9a-f]{64}", row["report_sha256"]) + or type(row.get("word_count")) is not int + or row["word_count"] < 0 + ): + raise ValueError(f"invalid canonical report index row for {run_id}") + attempt = row["attempt"] + directory = COLLECTION / "attempts" / run_id / str(attempt) + report = directory / "report.md" + attestation = json.loads((directory / "attestation.json").read_text()) + if attestation != row: + raise ValueError(f"indexed attestation mismatch for {run_id}") + if sha256_file(report) != row["report_sha256"]: + raise ValueError(f"indexed report hash mismatch for {run_id}") + text = report.read_bytes().decode("utf-8") + if len(text.split()) != row["word_count"]: + raise ValueError(f"indexed report word-count mismatch for {run_id}") + preserved = [ + event + for event in event_records() + if event["event"] == "report_preserved" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + ] + if len(preserved) != 1 or preserved[0].get("sha256") != row["report_sha256"]: + raise ValueError(f"canonical report lacks its preservation event: {run_id}") + attempts = [ + event + for event in event_records() + if event["event"] == "attempt_preserved" + and event.get("run_id") == run_id + and event.get("attempt") == attempt + ] + if len(attempts) != 1 or attempts[0].get("sha256") != byte_tree_digest(directory): + raise ValueError(f"canonical attempt differs from its preservation event: {run_id}") + return result + + +def presentation_order(mode: str, scorer: str) -> list[str]: + claim = f"{mode}-{scorer}" + rows = { + row["claim"]: row + for row in read_frozen_tsv(SEALED / "presentation-orders.tsv") + } + return rows[claim]["labels_in_order"].split(",") + + +def build_score_packets() -> None: + validate_static(require_lock=True, announce=False) + index = load_index() + if set(index) != set(load_schedule()): + raise ValueError("collection does not contain exactly all 80 scheduled reports") + if any(event["event"] == "collection_locked" for event in event_records()): + raise ValueError("collection was already locked") + append_event( + "collection", + "collection_locked", + digest=sha256_file(COLLECTION / "valid-index.jsonl"), + details={"report_count": 80}, + ) + (COLLECTION / "valid-index.jsonl").chmod(0o444) + verify_collection_lock() + blind = load_blind_map() + target_by_mode = {row["mode"]: row for row in load_target_map().values()} + packet_root = SCORING / "packets" + if packet_root.exists(): + raise FileExistsError("scorer packets already exist") + packet_digests: dict[str, str] = {} + for mode in MODES: + target_source = EVALS / target_by_mode[mode]["source_path"] + for scorer in SCORERS: + packet = packet_root / mode / scorer + (packet / "reports").mkdir(parents=True) + shutil.copytree(target_source, packet / "target") + shutil.copy2(FREEZE / "allowlists" / f"{mode}.txt", packet / "allowlist.txt") + shutil.copy2(FREEZE / "rubrics" / "SCORER.md", packet / "SCORER.md") + shutil.copy2(FREEZE / "rubrics" / f"{mode}.md", packet / "RUBRIC.md") + shutil.copy2(FREEZE / "schemas" / "score.schema.json", packet / "score.schema.json") + if tar_tree_digest(packet / "target") != target_by_mode[mode]["tree_sha256"]: + raise ValueError(f"copied target lost its frozen identity: {mode}/{scorer}") + frozen_copies = { + packet / "allowlist.txt": FREEZE / "allowlists" / f"{mode}.txt", + packet / "SCORER.md": FREEZE / "rubrics" / "SCORER.md", + packet / "RUBRIC.md": FREEZE / "rubrics" / f"{mode}.md", + packet / "score.schema.json": FREEZE / "schemas" / "score.schema.json", + } + for destination, source in frozen_copies.items(): + if sha256_file(destination) != frozen_file_digest(source): + raise ValueError(f"frozen packet input changed during copy: {destination}") + report_hashes: dict[str, str] = {} + for label, run_id in blind[mode].items(): + attempt = index[run_id]["attempt"] + source = COLLECTION / "attempts" / run_id / str(attempt) / "report.md" + if sha256_file(source) != index[run_id]["report_sha256"]: + raise ValueError(f"report changed before packet build: {run_id}") + destination = packet / "reports" / f"{label}.md" + shutil.copy2(source, destination) + report_hashes[label] = sha256_file(destination) + if report_hashes[label] != index[run_id]["report_sha256"]: + raise ValueError(f"report changed during packet copy: {run_id}") + manifest = { + "schema_version": 1, + "mode": mode, + "scorer_id": scorer, + "presentation_order": presentation_order(mode, scorer), + "report_sha256": report_hashes, + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "score.schema.json"), + } + write_once(packet / "PACKET.json", json_dump(manifest)) + make_read_only(packet) + packet_digest = byte_tree_digest(packet) + packet_digests[f"{mode}-{scorer}"] = packet_digest + append_event( + "scoring", + "blind_packet_preserved", + digest=packet_digest, + details={"mode": mode, "scorer": scorer}, + ) + digest = sha256_bytes(json.dumps(packet_digests, sort_keys=True).encode()) + append_event( + "scoring", + "blind_packets_built", + digest=digest, + details={"packet_count": 16}, + ) + print("built 16 blind scorer packets") + + +def verify_score_packet(mode: str, scorer: str) -> None: + verify_collection_lock() + packet = SCORING / "packets" / mode / scorer + manifest = json.loads((packet / "PACKET.json").read_text()) + if ( + type(manifest.get("schema_version")) is not int + or manifest["schema_version"] != 1 + or manifest.get("mode") != mode + or manifest.get("scorer_id") != scorer + ): + raise ValueError(f"score packet identity mismatch: {mode}/{scorer}") + if manifest.get("presentation_order") != presentation_order(mode, scorer): + raise ValueError(f"score packet presentation mismatch: {mode}/{scorer}") + checks = { + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "score.schema.json"), + } + for field, actual in checks.items(): + if manifest.get(field) != actual: + raise ValueError(f"score packet changed: {mode}/{scorer}/{field}") + report_hashes = { + label: sha256_file(packet / "reports" / f"{label}.md") for label in LABELS + } + if manifest.get("report_sha256") != report_hashes: + raise ValueError(f"score packet reports changed: {mode}/{scorer}") + expected_files = { + "PACKET.json", + "allowlist.txt", + "RUBRIC.md", + "SCORER.md", + "score.schema.json", + *(f"reports/{label}.md" for label in LABELS), + } + expected_files.update( + path.relative_to(packet).as_posix() + for path in (packet / "target").rglob("*") + if path.is_file() + ) + actual_files = { + path.relative_to(packet).as_posix() for path in packet.rglob("*") if path.is_file() + } + if actual_files != expected_files: + raise ValueError(f"score packet has unexpected files: {mode}/{scorer}") + expected_packet_digest = preserved_digest( + "blind_packet_preserved", mode=mode, scorer=scorer + ) + if byte_tree_digest(packet) != expected_packet_digest: + raise ValueError(f"score packet differs from its external preservation event: {mode}/{scorer}") + + +def render_packet_prompt(template_name: str, packet: Path, output: Path, **values: str) -> str: + template = read_frozen_text(FREEZE / "prompts" / template_name) + match = re.search(r"```text\n(.*?)\n```", template, flags=re.DOTALL) + if not match: + raise ValueError(f"{template_name} missing text fence") + prompt = match.group(1).replace("[PACKET]", str(packet)).replace("[OUTPUT]", str(output)) + for key, value in values.items(): + prompt = prompt.replace(f"[{key}]", value) + if re.search(r"\[[A-Z_]+\]", prompt): + raise ValueError(f"unresolved placeholder in {template_name}") + return prompt + + +def expected_evaluator_output(kind: str, identity: str, attempt: int) -> Path: + if attempt < 1: + raise ValueError("evaluator attempt must be positive") + seeds = load_frozen_seeds() + seed_name = "scorer" if kind == "scorer" else "presentation" + token = prepare.keyed( + f"{kind}-output-v2", seeds[seed_name], f"{identity}|{attempt}" + )[:32] + return Path("/tmp/ur-eval") / token / "output" + + +def expected_evaluator_packet(kind: str, identity: str, attempt: int) -> Path: + return expected_evaluator_output(kind, identity, attempt).parent / "packet" + + +def expected_source_packet_digest(kind: str, identity: str) -> str: + if kind == "scorer": + mode, scorer = identity.split("-", 1) + return preserved_digest("blind_packet_preserved", mode=mode, scorer=scorer) + return preserved_digest("adjudication_packet_preserved", mode=identity) + + +def prepare_evaluator_runtime( + kind: str, identity: str, attempt: int, source_packet: Path, output: Path +) -> Path: + assert_evaluator_attempt_allowed(kind, identity, attempt) + expected = expected_evaluator_output(kind, identity, attempt) + if output != expected: + raise ValueError(f"{kind} output must be the frozen neutral path {expected}") + root = output.parent + if root.exists(): + verify_evaluator_runtime(kind, identity, attempt, source_packet, output) + if any(output.iterdir()): + raise ValueError(f"existing {kind} prelaunch output is not empty") + return expected_evaluator_packet(kind, identity, attempt) + runtime_packet = expected_evaluator_packet(kind, identity, attempt) + root.mkdir(parents=True) + shutil.copytree(source_packet, runtime_packet) + if byte_tree_digest(runtime_packet) != expected_source_packet_digest(kind, identity): + raise ValueError(f"{kind} runtime packet differs from frozen packet") + make_read_only(runtime_packet) + output.mkdir() + os.utime(root, (0, 0), follow_symlinks=False) + root.chmod(0o555) + return runtime_packet + + +def verify_evaluator_runtime( + kind: str, + identity: str, + attempt: int, + source_packet: Path, + output: Path, + *, + allow_invalid_output: bool = False, +) -> None: + if output != expected_evaluator_output(kind, identity, attempt): + raise ValueError(f"non-neutral {kind} output path") + root = output.parent + if root.is_symlink() or not root.is_dir() or root.stat().st_mode & 0o222: + raise ValueError(f"invalid {kind} runtime root") + actual_entries = {entry.name for entry in root.iterdir()} + if ( + (not allow_invalid_output and actual_entries != {"packet", "output"}) + or ( + allow_invalid_output + and ("packet" not in actual_entries or actual_entries - {"packet", "output"}) + ) + ): + raise ValueError(f"unexpected {kind} runtime inventory") + if not allow_invalid_output and ( + not output.exists() + or output.is_symlink() + or not stat.S_ISDIR(output.lstat().st_mode) + ): + raise ValueError(f"invalid {kind} output directory") + runtime_packet = expected_evaluator_packet(kind, identity, attempt) + if byte_tree_digest(source_packet) != expected_source_packet_digest(kind, identity): + raise ValueError(f"{kind} source packet changed") + if byte_tree_digest(runtime_packet) != expected_source_packet_digest(kind, identity): + raise ValueError(f"{kind} runtime packet changed") + + +def evaluator_runtime_forensics( + kind: str, + identity: str, + attempt: int, + source_packet: Path, + output: Path, + error: Exception, +) -> dict[str, Any]: + expected_output = expected_evaluator_output(kind, identity, attempt) + if output != expected_output: + raise ValueError(f"forensic evaluator output is not the neutral path {expected_output}") + root = output.parent + root_is_real = root.exists() and not root.is_symlink() and root.is_dir() + return { + "schema_version": 1, + "kind": kind, + "identity": identity, + "attempt": attempt, + "verification_error": f"{type(error).__name__}: {error}", + "expected": { + "output": str(expected_output), + "source_packet_byte_tree_sha256": expected_source_packet_digest(kind, identity), + }, + "observed": { + "source_packet": observe_path(source_packet), + "runtime_root": observe_path(root), + "packet": observe_path(root / "packet") + if root_is_real + else {"type": "unavailable"}, + "output": observe_path(output) + if root_is_real + else {"type": "unavailable"}, + }, + } + + +def validate_score(value: Any, mode: str, scorer: str) -> None: + if ( + not isinstance(value, dict) + or type(value.get("schema_version")) is not int + or value["schema_version"] != 1 + ): + raise ValueError("invalid score envelope") + if set(value) != {"schema_version", "mode", "scorer_id", "reports", "ambiguities"}: + raise ValueError("unexpected score fields") + if value.get("mode") != mode or value.get("scorer_id") != scorer: + raise ValueError("score mode or scorer mismatch") + if not isinstance(value["ambiguities"], list) or any( + not is_nonblank_string(item) for item in value["ambiguities"] + ): + raise ValueError("invalid score ambiguities") + reports = value.get("reports") + if ( + not isinstance(reports, list) + or len(reports) != 10 + or any(not isinstance(report, dict) for report in reports) + ): + raise ValueError("score must contain ten reports") + report_labels = [report.get("label") for report in reports] + if any(not isinstance(label, str) for label in report_labels) or tuple( + sorted(report_labels) + ) != LABELS: + raise ValueError("score labels must be exactly A-J") + expected_atoms = set(atom_ids(mode)) + allowed_hard = set(hard_error_ids(mode)) + for report in reports: + if set(report) != { + "label", + "atoms", + "hard_errors", + "proposal_laundering", + "tcb_authority_defect", + "visible_scope_defect", + "novel_findings", + }: + raise ValueError("unexpected report-score fields") + atoms = report.get("atoms") + if ( + not isinstance(atoms, list) + or any(not isinstance(atom, dict) for atom in atoms) + or {atom.get("id") for atom in atoms} != expected_atoms + ): + raise ValueError(f"{mode}/{report.get('label')}: incomplete atom set") + if len(atoms) != len(expected_atoms): + raise ValueError(f"{mode}/{report.get('label')}: duplicate atom ID") + for atom in atoms: + if set(atom) != {"id", "decision", "evidence"}: + raise ValueError("unexpected atom fields") + if ( + atom.get("decision") not in {"PASS", "FAIL"} + or not is_nonblank_string(atom.get("evidence")) + ): + raise ValueError("invalid atom decision") + hard = report.get("hard_errors") + if not isinstance(hard, list) or any( + not isinstance(finding, dict) for finding in hard + ): + raise ValueError("hard_errors must be a list") + hard_ids = [finding.get("id") for finding in hard] + if len(hard_ids) != len(set(hard_ids)) or not set(hard_ids) <= allowed_hard: + raise ValueError(f"invalid hard-error IDs: {hard_ids}") + if any( + not is_nonblank_string(finding.get("evidence")) + for finding in hard + ): + raise ValueError("hard error lacks evidence") + if any(set(finding) != {"id", "evidence"} for finding in hard): + raise ValueError("unexpected hard-error fields") + for field in ("proposal_laundering", "tcb_authority_defect", "visible_scope_defect"): + flag = report.get(field) + if not isinstance(flag, dict) or not isinstance(flag.get("present"), bool): + raise ValueError(f"invalid {field} flag") + if set(flag) != {"present", "evidence"} or not is_nonblank_string( + flag.get("evidence") + ): + raise ValueError(f"invalid {field} evidence") + novel = report.get("novel_findings") + if not isinstance(novel, list) or any( + not isinstance(finding, dict) for finding in novel + ): + raise ValueError("novel_findings must be a list") + novel_ids = [finding.get("id") for finding in novel] + if len(novel_ids) != len(set(novel_ids)) or any( + not re.fullmatch(r"N[1-9][0-9]*", str(identifier)) for identifier in novel_ids + ): + raise ValueError("invalid novel-finding IDs") + if any( + not is_nonblank_string(finding.get("evidence")) + for finding in novel + ): + raise ValueError("novel finding lacks evidence") + if any(set(finding) != {"id", "evidence"} for finding in novel): + raise ValueError("unexpected novel-finding fields") + + +def record_score( + mode: str, scorer: str, attempt: int, output: Path, agent_id: str +) -> None: + assert_evaluator_started("scorer", f"{mode}-{scorer}", attempt, agent_id) + verify_score_packet(mode, scorer) + verify_evaluator_runtime( + "scorer", + f"{mode}-{scorer}", + attempt, + SCORING / "packets" / mode / scorer, + output, + ) + source = output / "score.json" + entries = output_entries(output) + if [entry.name for entry in entries] != ["score.json"]: + raise ValueError("scorer output is not exactly score.json") + raw = source.read_bytes() + value = json.loads(raw.decode("utf-8")) + validate_score(value, mode, scorer) + attempt_directory = ( + SCORING + / "evaluator-attempts" + / "scoring" + / f"{mode}-{scorer}" + / str(attempt) + ) + entries = preserve_captured_file(attempt_directory / "raw-output", "score.json", raw) + write_once(attempt_directory / "raw-output-manifest.json", json_dump(entries)) + attempt_attestation = { + "schema_version": 1, + "phase": "scoring", + "kind": "scorer", + "identity": f"{mode}-{scorer}", + "attempt": attempt, + "agent_id": agent_id, + "disposition": "COMPLETE", + "recorded_utc": utc_now(), + } + write_once(attempt_directory / "attestation.json", json_dump(attempt_attestation)) + make_read_only(attempt_directory) + append_event( + "scoring", + "evaluator_attempt_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(attempt_directory), + details={"kind": "scorer", "identity": f"{mode}-{scorer}", "disposition": "COMPLETE"}, + ) + destination = SCORING / "raw" / mode / f"{scorer}.json" + write_bytes_once(destination, raw) + destination.chmod(0o444) + append_event( + "scoring", + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": "scorer", "identity": f"{mode}-{scorer}", "api_state": "COMPLETED"}, + ) + append_event( + "scoring", + "score_preserved", + attempt=attempt, + agent_id=agent_id, + digest=sha256_file(destination), + details={"mode": mode, "scorer": scorer}, + ) + print(destination) + + +def report_by_label(score: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {report["label"]: report for report in score["reports"]} + + +def preserved_digest(event_name: str, **details: str) -> str: + matches = [ + event + for event in event_records() + if event["event"] == event_name + and all(event.get("details", {}).get(key) == value for key, value in details.items()) + ] + if len(matches) != 1 or "sha256" not in matches[0]: + raise ValueError(f"expected one preserved digest for {event_name} {details}") + return matches[0]["sha256"] + + +def load_preserved_score(mode: str, scorer: str) -> dict[str, Any]: + path = SCORING / "raw" / mode / f"{scorer}.json" + expected = preserved_digest("score_preserved", mode=mode, scorer=scorer) + if sha256_file(path) != expected: + raise ValueError(f"raw score changed after preservation: {mode}/{scorer}") + value = json.loads(path.read_text()) + validate_score(value, mode, scorer) + return value + + +def evidence_for_hard(report: dict[str, Any], identifier: str) -> str: + for finding in report["hard_errors"]: + if finding["id"] == identifier: + return finding["evidence"] + return "No applicable hard error recorded." + + +def disagreement_cells(mode: str, s1: dict[str, Any], s2: dict[str, Any]) -> list[dict[str, Any]]: + cells: list[dict[str, Any]] = [] + by_scorer = {"s1": report_by_label(s1), "s2": report_by_label(s2)} + for label in LABELS: + first = by_scorer["s1"][label] + second = by_scorer["s2"][label] + atoms1 = {atom["id"]: atom for atom in first["atoms"]} + atoms2 = {atom["id"]: atom for atom in second["atoms"]} + for atom in atom_ids(mode): + if atoms1[atom]["decision"] != atoms2[atom]["decision"]: + cells.append( + { + "label": label, + "field": f"atom:{atom}", + "s1": {"decision": atoms1[atom]["decision"], "evidence": atoms1[atom]["evidence"]}, + "s2": {"decision": atoms2[atom]["decision"], "evidence": atoms2[atom]["evidence"]}, + } + ) + for identifier in hard_error_ids(mode): + present1 = any(item["id"] == identifier for item in first["hard_errors"]) + present2 = any(item["id"] == identifier for item in second["hard_errors"]) + if present1 != present2: + cells.append( + { + "label": label, + "field": f"hard_error:{identifier}", + "s1": {"decision": "PRESENT" if present1 else "ABSENT", "evidence": evidence_for_hard(first, identifier)}, + "s2": {"decision": "PRESENT" if present2 else "ABSENT", "evidence": evidence_for_hard(second, identifier)}, + } + ) + for field in ("proposal_laundering", "tcb_authority_defect", "visible_scope_defect"): + flag1 = first[field] + flag2 = second[field] + if flag1["present"] != flag2["present"]: + cells.append( + { + "label": label, + "field": field, + "s1": {"decision": "PRESENT" if flag1["present"] else "ABSENT", "evidence": flag1["evidence"]}, + "s2": {"decision": "PRESENT" if flag2["present"] else "ABSENT", "evidence": flag2["evidence"]}, + } + ) + for scorer, report, other in (("s1", first, "s2"), ("s2", second, "s1")): + for finding in report["novel_findings"]: + cells.append( + { + "label": label, + "field": f"novel:{scorer}:{finding['id']}", + scorer: {"decision": "PRESENT", "evidence": finding["evidence"]}, + other: {"decision": "ABSENT", "evidence": "Not independently proposed; adjudicate the candidate on its merits."}, + } + ) + keys = [(cell["label"], cell["field"]) for cell in cells] + if len(keys) != len(set(keys)): + raise ValueError("duplicate disagreement field") + return cells + + +def build_disagreements(mode: str) -> None: + scores: dict[str, dict[str, Any]] = {} + for scorer in SCORERS: + scores[scorer] = load_preserved_score(mode, scorer) + value = {"schema_version": 1, "mode": mode, "cells": disagreement_cells(mode, scores["s1"], scores["s2"])} + destination = SCORING / "disagreements" / f"{mode}.json" + write_once(destination, json_dump(value)) + destination.chmod(0o444) + append_event( + "adjudication", + "disagreements_materialized", + digest=sha256_file(destination), + details={"mode": mode, "count": len(value["cells"])}, + ) + print(f"{mode}: {len(value['cells'])} disputed or novel cells") + + +def build_adjudication_packet(mode: str) -> None: + disagreement_path = SCORING / "disagreements" / f"{mode}.json" + if sha256_file(disagreement_path) != preserved_digest("disagreements_materialized", mode=mode): + raise ValueError(f"disagreements changed before packet build: {mode}") + disagreements = json.loads(disagreement_path.read_text()) + if not disagreements["cells"]: + print(f"{mode}: no adjudication packet required") + return + packet = SCORING / "adjudication-packets" / mode + if packet.exists(): + raise FileExistsError(f"adjudication packet exists: {packet}") + (packet / "reports").mkdir(parents=True) + source_packet = SCORING / "packets" / mode / "s1" + verify_score_packet(mode, "s1") + source_packet_digest = preserved_digest("blind_packet_preserved", mode=mode, scorer="s1") + source_manifest = json.loads((source_packet / "PACKET.json").read_text()) + shutil.copytree(source_packet / "target", packet / "target") + for name in ("allowlist.txt", "SCORER.md", "RUBRIC.md"): + shutil.copy2(source_packet / name, packet / name) + shutil.copy2(FREEZE / "schemas" / "adjudication.schema.json", packet / "adjudication.schema.json") + shutil.copy2(disagreement_path, packet / "DISAGREEMENTS.json") + disputed_labels = sorted({cell["label"] for cell in disagreements["cells"]}) + for label in disputed_labels: + shutil.copy2(source_packet / "reports" / f"{label}.md", packet / "reports" / f"{label}.md") + if ( + sha256_file(packet / "reports" / f"{label}.md") + != source_manifest["report_sha256"][label] + ): + raise ValueError(f"adjudication report changed during copy: {mode}/{label}") + target_row = next(row for row in load_target_map().values() if row["mode"] == mode) + if tar_tree_digest(packet / "target") != target_row["tree_sha256"]: + raise ValueError(f"adjudication target lost frozen identity: {mode}") + if sha256_file(packet / "DISAGREEMENTS.json") != preserved_digest( + "disagreements_materialized", mode=mode + ): + raise ValueError(f"adjudication disagreements changed during copy: {mode}") + expected_copies = { + "allowlist.txt": "allowlist_sha256", + "SCORER.md": "common_rules_sha256", + "RUBRIC.md": "rubric_sha256", + } + for name, field in expected_copies.items(): + if sha256_file(packet / name) != source_manifest[field]: + raise ValueError(f"adjudication packet input changed during copy: {mode}/{name}") + if sha256_file(packet / "adjudication.schema.json") != frozen_file_digest( + FREEZE / "schemas" / "adjudication.schema.json" + ): + raise ValueError(f"adjudication schema changed during copy: {mode}") + manifest = { + "schema_version": 1, + "mode": mode, + "source_score_packet_sha256": source_packet_digest, + "disputed_labels": disputed_labels, + "disagreements_sha256": sha256_file(packet / "DISAGREEMENTS.json"), + "report_sha256": {label: sha256_file(packet / "reports" / f"{label}.md") for label in disputed_labels}, + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "adjudication.schema.json"), + } + write_once(packet / "PACKET.json", json_dump(manifest)) + make_read_only(packet) + append_event( + "adjudication", + "adjudication_packet_preserved", + digest=byte_tree_digest(packet), + details={"mode": mode, "source_score_packet_sha256": source_packet_digest}, + ) + print(packet) + + +def verify_adjudication_packet(mode: str) -> None: + packet = SCORING / "adjudication-packets" / mode + manifest = json.loads((packet / "PACKET.json").read_text()) + disagreements = json.loads((packet / "DISAGREEMENTS.json").read_text()) + if ( + type(manifest.get("schema_version")) is not int + or manifest["schema_version"] != 1 + or manifest.get("mode") != mode + or disagreements.get("mode") != mode + ): + raise ValueError(f"adjudication packet identity mismatch: {mode}") + verify_score_packet(mode, "s1") + source_digest = preserved_digest("blind_packet_preserved", mode=mode, scorer="s1") + if manifest.get("source_score_packet_sha256") != source_digest: + raise ValueError(f"adjudication packet source binding changed: {mode}") + checks = { + "disagreements_sha256": sha256_file(packet / "DISAGREEMENTS.json"), + "target_byte_tree_sha256": byte_tree_digest(packet / "target"), + "allowlist_sha256": sha256_file(packet / "allowlist.txt"), + "common_rules_sha256": sha256_file(packet / "SCORER.md"), + "rubric_sha256": sha256_file(packet / "RUBRIC.md"), + "schema_sha256": sha256_file(packet / "adjudication.schema.json"), + } + for field, actual in checks.items(): + if manifest.get(field) != actual: + raise ValueError(f"adjudication packet changed: {mode}/{field}") + labels = manifest.get("disputed_labels") + expected_labels = sorted({cell["label"] for cell in disagreements.get("cells", [])}) + if not isinstance(labels, list) or labels != expected_labels: + raise ValueError(f"adjudication packet disputed-label inventory changed: {mode}") + report_hashes = { + label: sha256_file(packet / "reports" / f"{label}.md") for label in labels + } + if manifest.get("report_sha256") != report_hashes: + raise ValueError(f"adjudication reports changed: {mode}") + expected_files = { + "PACKET.json", + "DISAGREEMENTS.json", + "allowlist.txt", + "RUBRIC.md", + "SCORER.md", + "adjudication.schema.json", + *(f"reports/{label}.md" for label in labels), + } + expected_files.update( + path.relative_to(packet).as_posix() + for path in (packet / "target").rglob("*") + if path.is_file() + ) + actual_files = { + path.relative_to(packet).as_posix() for path in packet.rglob("*") if path.is_file() + } + if actual_files != expected_files: + raise ValueError(f"adjudication packet has unexpected files: {mode}") + expected_packet_digest = preserved_digest("adjudication_packet_preserved", mode=mode) + if byte_tree_digest(packet) != expected_packet_digest: + raise ValueError(f"adjudication packet differs from its external event: {mode}") + + +def validate_adjudication(value: Any, mode: str) -> None: + disagreement_path = SCORING / "disagreements" / f"{mode}.json" + expected_digest = preserved_digest("disagreements_materialized", mode=mode) + if sha256_file(disagreement_path) != expected_digest: + raise ValueError(f"disagreements changed after preservation: {mode}") + disagreements = json.loads(disagreement_path.read_text()) + expected = {(cell["label"], cell["field"]) for cell in disagreements["cells"]} + if ( + not isinstance(value, dict) + or type(value.get("schema_version")) is not int + or value["schema_version"] != 1 + or value.get("mode") != mode + ): + raise ValueError("invalid adjudication envelope") + if set(value) != {"schema_version", "mode", "decisions", "ambiguities"}: + raise ValueError("unexpected adjudication fields") + if not isinstance(value["ambiguities"], list) or any( + not is_nonblank_string(item) for item in value["ambiguities"] + ): + raise ValueError("invalid adjudication ambiguities") + decisions = value.get("decisions") + if not isinstance(decisions, list) or any( + not isinstance(decision, dict) for decision in decisions + ): + raise ValueError("adjudication decisions must be a list") + if any( + not isinstance(decision.get("label"), str) + or not isinstance(decision.get("field"), str) + for decision in decisions + ): + raise ValueError("adjudication decision identity is invalid") + actual = {(decision.get("label"), decision.get("field")) for decision in decisions} + if actual != expected or len(decisions) != len(expected): + raise ValueError("adjudication does not resolve exactly every disputed cell") + for decision in decisions: + if set(decision) != {"label", "field", "decision", "evidence"}: + raise ValueError("unexpected adjudication-decision fields") + expected_values = {"PASS", "FAIL"} if decision["field"].startswith("atom:") else {"PRESENT", "ABSENT"} + if ( + decision.get("decision") not in expected_values + or not is_nonblank_string(decision.get("evidence")) + ): + raise ValueError("invalid adjudication decision") + + +def synthetic_score(mode: str, scorer: str) -> dict[str, Any]: + reports: list[dict[str, Any]] = [] + for label in LABELS: + reports.append( + { + "label": label, + "atoms": [ + {"id": atom, "decision": "PASS", "evidence": "Synthetic complete evidence."} + for atom in atom_ids(mode) + ], + "hard_errors": [], + "proposal_laundering": {"present": False, "evidence": "No proposal laundering."}, + "tcb_authority_defect": {"present": False, "evidence": "No TCB or authority defect."}, + "visible_scope_defect": {"present": False, "evidence": "No visible source-scope defect."}, + "novel_findings": [], + } + ) + return { + "schema_version": 1, + "mode": mode, + "scorer_id": scorer, + "reports": reports, + "ambiguities": [], + } + + +def self_test() -> None: + validate_static(require_lock=False) + for mode in MODES: + first = synthetic_score(mode, "s1") + second = synthetic_score(mode, "s2") + validate_score(first, mode, "s1") + validate_score(second, mode, "s2") + first_report = first["reports"][0] + second_report = second["reports"][0] + first_report["atoms"][0]["decision"] = "FAIL" + first_report["hard_errors"] = [ + {"id": hard_error_ids(mode)[0], "evidence": "Synthetic hard-error evidence."} + ] + first_report["proposal_laundering"] = {"present": True, "evidence": "Synthetic flag evidence."} + first_report["novel_findings"] = [{"id": "N1", "evidence": "Synthetic novel candidate."}] + cells = disagreement_cells(mode, first, second) + expected_fields = { + f"atom:{atom_ids(mode)[0]}", + f"hard_error:{hard_error_ids(mode)[0]}", + "proposal_laundering", + "novel:s1:N1", + } + if {cell["field"] for cell in cells} != expected_fields: + raise AssertionError(f"{mode}: disagreement self-test failed") + invalid = synthetic_score(mode, "s1") + invalid["reports"][1]["label"] = "A" + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: duplicate label was accepted") + invalid = synthetic_score(mode, "s1") + invalid["reports"][0]["atoms"].pop() + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: incomplete atom set was accepted") + invalid = synthetic_score(mode, "s1") + invalid["schema_version"] = True + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: boolean schema version was accepted") + invalid = synthetic_score(mode, "s1") + invalid["reports"][0]["atoms"][0]["evidence"] = 1 + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: non-string evidence was accepted") + invalid = synthetic_score(mode, "s1") + invalid["reports"][0]["atoms"][0]["evidence"] = " " + try: + validate_score(invalid, mode, "s1") + except ValueError: + pass + else: + raise AssertionError(f"{mode}: whitespace-only evidence was accepted") + schedule = load_schedule() + for run_id, row in schedule.items(): + runtime = Path("/tmp/ur-eval") / row["cell_id"] + prompt = render_report_prompt(run_id, runtime) + if "[" + "PACKAGE]" in prompt or "[" + "WORD_LIMIT]" in prompt: + raise AssertionError("unresolved report prompt") + if report_reminder_text() != ( + "Complete now within the frozen word limit using only material already\n" + "inspected; do not widen scope." + ): + raise AssertionError("frozen reminder extraction changed") + if expected_evaluator_output("scorer", "S-s1", 1) == expected_evaluator_output( + "scorer", "S-s1", 2 + ): + raise AssertionError("evaluator attempts share a runtime path") + assert_evaluator_attempt_allowed("scorer", "S-s1", 1, []) + try: + assert_evaluator_attempt_allowed("scorer", "S-s1", 2, []) + except ValueError: + pass + else: + raise AssertionError("evaluator retry without infrastructure failure was accepted") + assert_evaluator_attempt_allowed( + "scorer", + "S-s1", + 2, + [ + { + "event": "evaluator_infrastructure_failure", + "attempt": 1, + "details": {"kind": "scorer", "identity": "S-s1"}, + } + ], + ) + invalid_event = { + "schema_version": True, + "sequence": 1, + "previous_event_sha256": None, + "time_utc": utc_now(), + "phase": "freeze", + "event": "synthetic", + "details": {}, + } + try: + validate_event_record(invalid_event, 1) + except ValueError: + pass + else: + raise AssertionError("boolean event schema version was accepted") + metadata_root = Path(tempfile.mkdtemp(prefix="ur-packet-self-test-", dir="/tmp")) + try: + nested = metadata_root / "nested" + nested.mkdir() + file = nested / "report.md" + file.write_text("packet metadata self-test\n") + make_read_only(metadata_root) + for item in (metadata_root, nested, file): + if item.stat().st_mtime_ns != 0: + raise AssertionError(f"metadata timestamp was not normalized: {item}") + finally: + for item in sorted(metadata_root.rglob("*"), reverse=True): + item.chmod(0o755 if item.is_dir() else 0o644) + metadata_root.chmod(0o755) + shutil.rmtree(metadata_root) + print("protocol self-test passed") + + +def record_adjudication( + mode: str, attempt: int, output: Path, agent_id: str +) -> None: + assert_evaluator_started("adjudicator", mode, attempt, agent_id) + verify_adjudication_packet(mode) + verify_evaluator_runtime( + "adjudicator", + mode, + attempt, + SCORING / "adjudication-packets" / mode, + output, + ) + entries = output_entries(output) + if [entry.name for entry in entries] != ["adjudication.json"]: + raise ValueError("adjudicator output is not exactly adjudication.json") + raw = entries[0].read_bytes() + value = json.loads(raw.decode("utf-8")) + validate_adjudication(value, mode) + attempt_directory = ( + SCORING + / "evaluator-attempts" + / "adjudication" + / mode + / str(attempt) + ) + raw_entries = preserve_captured_file( + attempt_directory / "raw-output", "adjudication.json", raw + ) + write_once( + attempt_directory / "raw-output-manifest.json", json_dump(raw_entries) + ) + attempt_attestation = { + "schema_version": 1, + "phase": "adjudication", + "kind": "adjudicator", + "identity": mode, + "attempt": attempt, + "agent_id": agent_id, + "disposition": "COMPLETE", + "recorded_utc": utc_now(), + } + write_once(attempt_directory / "attestation.json", json_dump(attempt_attestation)) + make_read_only(attempt_directory) + append_event( + "adjudication", + "evaluator_attempt_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(attempt_directory), + details={"kind": "adjudicator", "identity": mode, "disposition": "COMPLETE"}, + ) + destination = SCORING / "adjudications" / f"{mode}.json" + write_bytes_once(destination, raw) + destination.chmod(0o444) + append_event( + "adjudication", + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": "adjudicator", "identity": mode, "api_state": "COMPLETED"}, + ) + append_event( + "adjudication", + "adjudication_preserved", + attempt=attempt, + agent_id=agent_id, + digest=sha256_file(destination), + details={"mode": mode}, + ) + print(destination) + + +def record_invalid_evaluator( + phase: str, + identity: str, + attempt: int, + output: Path, + agent_id: str, + evidence: str, +) -> None: + invalid_marker = RUN / "INVALID.json" + if invalid_marker.exists(): + raise FileExistsError("run is already marked INVALID") + if not evidence.strip(): + raise ValueError("invalid evaluator output requires nonempty evidence") + kind = "scorer" if phase == "scoring" else "adjudicator" + assert_evaluator_started(kind, identity, attempt, agent_id) + if kind == "scorer": + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + else: + source_packet = SCORING / "adjudication-packets" / identity + forensics: dict[str, Any] | None = None + try: + verify_evaluator_runtime( + kind, identity, attempt, source_packet, output, allow_invalid_output=True + ) + except (OSError, ValueError) as error: + forensics = evaluator_runtime_forensics( + kind, identity, attempt, source_packet, output, error + ) + evidence = ( + f"{evidence} Runtime/packet verification failed: " + f"{forensics['verification_error']}" + ) + destination = SCORING / "invalid" / phase / identity / str(attempt) + root = output.parent + if forensics is None: + entries = snapshot_path(output, destination / "raw-output") + write_once(destination / "raw-output-manifest.json", json_dump(entries)) + else: + entries = snapshot_path(root, destination / "raw-runtime") + write_once(destination / "raw-runtime-manifest.json", json_dump(entries)) + source_entries = snapshot_path( + source_packet, destination / "source-packet-at-verification" + ) + write_once( + destination / "source-packet-at-verification-manifest.json", + json_dump(source_entries), + ) + if forensics is not None: + write_once(destination / "runtime-forensics.json", json_dump(forensics)) + attestation = { + "schema_version": 1, + "phase": phase, + "identity": identity, + "attempt": attempt, + "agent_id": agent_id, + "evidence": evidence, + "disposition": "INVALID_NONRERUNNABLE_EVALUATOR_OUTPUT", + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(attestation)) + make_read_only(destination) + append_event( + phase, + "invalid_output_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"kind": kind, "identity": identity}, + ) + write_once(invalid_marker, json_dump(attestation)) + invalid_marker.chmod(0o444) + append_event( + phase, + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": kind, "identity": identity, "api_state": "INVALID_OUTPUT"}, + ) + append_event( + phase, + "run_invalidated", + attempt=attempt, + agent_id=agent_id, + digest=sha256_file(invalid_marker), + details={"identity": identity, "evidence": evidence}, + ) + print(invalid_marker) + + +def preserve_failed_evaluator_attempt( + phase: str, + identity: str, + attempt: int, + output: Path, + agent_id: str, + disposition: str, + evidence: str, +) -> None: + if disposition not in INFRA_FAILURE_CODES: + raise ValueError(f"non-infrastructure evaluator disposition: {disposition}") + if not evidence.strip(): + raise ValueError("evaluator infrastructure failure requires nonempty evidence") + kind = "scorer" if phase == "scoring" else "adjudicator" + assert_evaluator_started(kind, identity, attempt, agent_id) + if kind == "scorer": + mode, scorer = identity.split("-", 1) + source_packet = SCORING / "packets" / mode / scorer + else: + source_packet = SCORING / "adjudication-packets" / identity + verify_evaluator_runtime( + kind, identity, attempt, source_packet, output, allow_invalid_output=True + ) + destination = SCORING / "evaluator-attempts" / phase / identity / str(attempt) + entries = snapshot_path(output, destination / "raw-output") + write_once(destination / "raw-output-manifest.json", json_dump(entries)) + attestation = { + "schema_version": 1, + "phase": phase, + "kind": kind, + "identity": identity, + "attempt": attempt, + "agent_id": agent_id, + "evidence": evidence, + "disposition": disposition, + "recorded_utc": utc_now(), + } + write_once(destination / "attestation.json", json_dump(attestation)) + make_read_only(destination) + append_event( + phase, + "evaluator_attempt_preserved", + attempt=attempt, + agent_id=agent_id, + digest=byte_tree_digest(destination), + details={"kind": kind, "identity": identity, "disposition": disposition}, + ) + append_event( + phase, + "evaluator_returned", + attempt=attempt, + agent_id=agent_id, + details={"kind": kind, "identity": identity, "api_state": "INFRASTRUCTURE_FAILURE"}, + ) + append_event( + phase, + "evaluator_infrastructure_failure", + attempt=attempt, + agent_id=agent_id, + details={ + "kind": kind, + "identity": identity, + "disposition": disposition, + "evidence": evidence, + }, + ) + print(json_dump(attestation), end="") + + +def decision_lookup(value: dict[str, Any]) -> dict[tuple[str, str], dict[str, Any]]: + return {(item["label"], item["field"]): item for item in value.get("decisions", [])} + + +def merge_final(mode: str) -> None: + scores = {scorer: load_preserved_score(mode, scorer) for scorer in SCORERS} + disagreement_path = SCORING / "disagreements" / f"{mode}.json" + if sha256_file(disagreement_path) != preserved_digest( + "disagreements_materialized", mode=mode + ): + raise ValueError(f"disagreements changed after preservation: {mode}") + disagreements = json.loads(disagreement_path.read_text()) + if disagreements["cells"]: + adjudication = json.loads((SCORING / "adjudications" / f"{mode}.json").read_text()) + expected_adjudication = preserved_digest("adjudication_preserved", mode=mode) + if sha256_file(SCORING / "adjudications" / f"{mode}.json") != expected_adjudication: + raise ValueError(f"adjudication changed after preservation: {mode}") + validate_adjudication(adjudication, mode) + decisions = decision_lookup(adjudication) + else: + decisions = {} + by_scorer = {scorer: report_by_label(score) for scorer, score in scores.items()} + final_reports: list[dict[str, Any]] = [] + confirmed_novel: list[dict[str, Any]] = [] + for label in LABELS: + first = by_scorer["s1"][label] + second = by_scorer["s2"][label] + atoms: dict[str, str] = {} + atoms1 = {atom["id"]: atom for atom in first["atoms"]} + atoms2 = {atom["id"]: atom for atom in second["atoms"]} + for atom in atom_ids(mode): + if atoms1[atom]["decision"] == atoms2[atom]["decision"]: + atoms[atom] = atoms1[atom]["decision"] + else: + atoms[atom] = decisions[(label, f"atom:{atom}")]["decision"] + final_hard: list[str] = [] + for identifier in hard_error_ids(mode): + present = [any(item["id"] == identifier for item in report["hard_errors"]) for report in (first, second)] + if present[0] == present[1]: + chosen = present[0] + else: + chosen = decisions[(label, f"hard_error:{identifier}")]["decision"] == "PRESENT" + if chosen: + final_hard.append(identifier) + flags: dict[str, bool] = {} + for field in ("proposal_laundering", "tcb_authority_defect", "visible_scope_defect"): + present = [first[field]["present"], second[field]["present"]] + if present[0] == present[1]: + flags[field] = present[0] + else: + flags[field] = decisions[(label, field)]["decision"] == "PRESENT" + final_reports.append({"label": label, "atoms": atoms, "hard_errors": final_hard, **flags}) + for scorer, report in (("s1", first), ("s2", second)): + for finding in report["novel_findings"]: + field = f"novel:{scorer}:{finding['id']}" + if decisions[(label, field)]["decision"] == "PRESENT": + confirmed_novel.append({"label": label, "field": field, "evidence": decisions[(label, field)]["evidence"]}) + value = {"schema_version": 1, "mode": mode, "reports": final_reports, "confirmed_novel_findings": confirmed_novel} + destination = SCORING / "final" / f"{mode}.json" + write_once(destination, json_dump(value)) + destination.chmod(0o444) + append_event("adjudication", "final_blind_score_locked", digest=sha256_file(destination), details={"mode": mode}) + print(destination) + + +def aggregate() -> None: + schedule = load_schedule() + conditions = load_condition_map() + targets = load_target_map() + blind = load_blind_map() + index = load_index(require_collection_lock=True) + counts: dict[str, dict[str, dict[str, int]]] = {} + defects: dict[str, dict[str, list[dict[str, Any]]]] = {} + failures: list[dict[str, Any]] = [] + novel: list[dict[str, Any]] = [] + final_digests: dict[str, str] = {} + for mode in MODES: + path = SCORING / "final" / f"{mode}.json" + expected_final = preserved_digest("final_blind_score_locked", mode=mode) + if sha256_file(path) != expected_final: + raise ValueError(f"final blind score changed after lock: {mode}") + value = json.loads(path.read_text()) + final_digests[mode] = sha256_file(path) + reports = {report["label"]: report for report in value["reports"]} + if tuple(sorted(reports)) != LABELS: + raise ValueError(f"{mode} final labels incomplete") + counts[mode] = {role: {atom: 0 for atom in atom_ids(mode)} for role in prepare.CONDITIONS} + defects[mode] = {role: [] for role in prepare.CONDITIONS} + for label, run_id in blind[mode].items(): + row = schedule[run_id] + if targets[row["target_label"]]["mode"] != mode: + raise ValueError("unblinding mode mismatch") + role = conditions[row["condition_label"]]["role"] + report = reports[label] + for atom, decision in report["atoms"].items(): + if decision == "PASS": + counts[mode][role][atom] += 1 + else: + failures.append({"mode": mode, "role": role, "run_id": run_id, "label": label, "atom": atom}) + over_budget = index[run_id]["word_count"] > index[run_id]["word_cap"] + flags = { + "hard_errors": report["hard_errors"], + "proposal_laundering": report["proposal_laundering"], + "tcb_authority_defect": report["tcb_authority_defect"], + "visible_scope_defect": report["visible_scope_defect"], + "operational_scope_deviation": index[run_id]["operational_scope_deviation"], + "word_budget_defect": over_budget, + "semantic_noncompletion": index[run_id]["semantic_noncompletion"], + } + if flags["hard_errors"] or any( + flags[name] + for name in ( + "proposal_laundering", + "tcb_authority_defect", + "visible_scope_defect", + "operational_scope_deviation", + "word_budget_defect", + "semantic_noncompletion", + ) + ): + defects[mode][role].append({"run_id": run_id, "label": label, **flags}) + for finding in value["confirmed_novel_findings"]: + novel.append({"mode": mode, **finding}) + v3_failures = [failure for failure in failures if failure["role"] == "v3"] + v3_defects = [item for mode in MODES for item in defects[mode]["v3"]] + gate = { + "all_v3_atoms_5_of_5": not v3_failures, + "zero_v3_hard_errors": not any(item["hard_errors"] for item in v3_defects), + "zero_v3_proposal_laundering": not any(item["proposal_laundering"] for item in v3_defects), + "zero_v3_tcb_authority_defects": not any(item["tcb_authority_defect"] for item in v3_defects), + "zero_v3_semantic_noncompletion": not any( + item["semantic_noncompletion"] for item in v3_defects + ), + "zero_v3_scope_budget_defects": not any( + item["visible_scope_defect"] + or item["operational_scope_deviation"] + or item["word_budget_defect"] + for item in v3_defects + ), + } + gate["overall"] = all(gate.values()) + diagnostic_comparisons: list[dict[str, Any]] = [] + for mode in MODES: + for atom in atom_ids(mode): + v3_count = counts[mode]["v3"][atom] + v2_count = counts[mode]["v2"][atom] + if v3_count < v2_count: + classification = "V3_BELOW_V2" + elif v3_count == 5 and v2_count < 5: + classification = "TARGETED_LIFT_EVIDENCE" + elif v3_count == 5 and v2_count == 5: + classification = "CEILING_REPLICATION" + elif v3_count > v2_count: + classification = "V3_HIGHER_BUT_CONFIRMATION_FAILED" + else: + classification = "MATCHED_BELOW_CEILING" + diagnostic_comparisons.append( + { + "mode": mode, + "atom": atom, + "v3_passes": v3_count, + "v2_passes": v2_count, + "classification": classification, + } + ) + result = { + "schema_version": 1, + "unblinded_utc": utc_now(), + "condition_map": {label: row["role"] for label, row in conditions.items()}, + "final_score_sha256": final_digests, + "counts": counts, + "defects": defects, + "failed_atom_cells": failures, + "confirmed_novel_findings": novel, + "diagnostic_comparison": { + "any_v3_below_v2": any( + item["classification"] == "V3_BELOW_V2" + for item in diagnostic_comparisons + ), + "causal_claim": False, + "reason": "The coherent V3 and V2 packages differ in more than one isolated instruction.", + "atoms": diagnostic_comparisons, + }, + "v3_gate": gate, + } + RESULTS.mkdir(parents=True, exist_ok=True) + write_once(RUN / "unblinding.json", json_dump({"schema_version": 1, "unblinded_utc": result["unblinded_utc"], "condition_map": result["condition_map"], "final_score_sha256": final_digests})) + write_once(RESULTS / "aggregate.json", json_dump(result)) + lines = [ + "# V3 Targeted Confirmation Results", + "", + f"**Primary V3 gate: {'PASS' if gate['overall'] else 'FAIL'}.**", + "", + "Each atom cell is a pass count out of five. V3 is the confirmatory candidate; V2 is diagnostic.", + "", + "| Mode | Condition | Atom pass counts | Defective reports |", + "|---|---|---|---:|", + ] + for mode in MODES: + for role in ("v3", "v2"): + rendered = "; ".join(f"{atom} {count}/5" for atom, count in counts[mode][role].items()) + lines.append(f"| {mode} | {role.upper()} | {rendered} | {len(defects[mode][role])} |") + lines.extend( + [ + "", + "## Diagnostic comparison", + "", + f"Any V3 atom below matched V2: {'YES' if result['diagnostic_comparison']['any_v3_below_v2'] else 'NO'}.", + "", + "`TARGETED_LIFT_EVIDENCE` means V3 passed 5/5 while matched V2 was lower. " + "`CEILING_REPLICATION` means both passed 5/5. These coherent packages differ " + "in more than one isolated instruction, so no classification is causal proof.", + "", + "| Mode | Atom | V3 | V2 | Classification |", + "|---|---|---:|---:|---|", + ] + ) + for item in diagnostic_comparisons: + lines.append( + f"| {item['mode']} | {item['atom']} | {item['v3_passes']}/5 | " + f"{item['v2_passes']}/5 | {item['classification']} |" + ) + lines.extend(["", "## Primary gates", "", "| Gate | Result |", "|---|---|"]) + for name, passed in gate.items(): + if name != "overall": + lines.append(f"| {name.replace('_', ' ')} | {'PASS' if passed else 'FAIL'} |") + lines.extend(["", "## Integrity limitations", "", "Filesystem and URL isolation were procedural on a shared host. Exact hosted model-build and sampling-seed metadata were unavailable. Results are source-review capability observations under those constraints.", ""]) + write_once(RESULTS / "summary.md", "\n".join(lines)) + append_event("unblinding", "conditions_revealed", digest=sha256_file(RUN / "unblinding.json"), details={}) + append_event( + "result", + "aggregate_written", + digest=sha256_file(RESULTS / "aggregate.json"), + details={ + "v3_gate": gate["overall"], + "summary_sha256": sha256_file(RESULTS / "summary.md"), + }, + ) + print(RESULTS / "summary.md") + + +def record_freeze_lock() -> None: + if any(event["event"] == "freeze_locked" for event in event_records()): + raise ValueError("freeze lock was already recorded") + append_event( + "freeze", + "freeze_locked", + digest=sha256_file(FILE_MANIFEST), + details={"lock_sha256": sha256_file(LOCK)}, + ) + print("freeze lock recorded") + + +def assert_freeze_locked() -> None: + matches = [event for event in event_records() if event["event"] == "freeze_locked"] + if len(matches) != 1: + raise ValueError("evaluation operations require exactly one freeze-lock event") + event = matches[0] + if ( + event.get("sha256") != sha256_file(FILE_MANIFEST) + or event.get("details", {}).get("lock_sha256") != sha256_file(LOCK) + ): + raise ValueError("freeze-lock event does not bind the current lock") + + +def main() -> None: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + verify = sub.add_parser("verify-static") + verify.add_argument("--locked", action="store_true") + sub.add_parser("self-test") + sub.add_parser("write-file-manifest") + sub.add_parser("record-freeze-lock") + prepare_cell_parser = sub.add_parser("prepare-cell") + prepare_cell_parser.add_argument("run_id") + prepare_cell_parser.add_argument("runtime", type=Path) + report_prompt = sub.add_parser("report-prompt") + report_prompt.add_argument("run_id") + report_prompt.add_argument("runtime", type=Path) + record_report_parser = sub.add_parser("record-report") + record_report_parser.add_argument("run_id") + record_report_parser.add_argument("attempt", type=positive_int) + record_report_parser.add_argument("runtime", type=Path) + record_report_parser.add_argument("agent_id") + record_report_parser.add_argument("--scope-deviation", action="store_true") + record_report_parser.add_argument( + "--scope-evidence", default="No known operational source-scope deviation." + ) + failed_report_parser = sub.add_parser("record-failed-report") + failed_report_parser.add_argument("run_id") + failed_report_parser.add_argument("attempt", type=positive_int) + failed_report_parser.add_argument("runtime", type=Path) + failed_report_parser.add_argument("agent_id") + failed_report_parser.add_argument( + "disposition", choices=sorted(INFRA_FAILURE_CODES | TERMINAL_REPORT_FAILURE_CODES) + ) + failed_report_parser.add_argument("evidence") + failed_report_parser.add_argument("--scope-deviation", action="store_true") + agent_start_parser = sub.add_parser("agent-start") + agent_start_parser.add_argument("run_id") + agent_start_parser.add_argument("attempt", type=positive_int) + agent_start_parser.add_argument("agent_id") + prelaunch_parser = sub.add_parser("record-prelaunch-failure") + prelaunch_parser.add_argument("run_id") + prelaunch_parser.add_argument("evidence") + reminder_parser = sub.add_parser("reminder-text") + reminder_parser.add_argument("run_id") + reminder_parser.add_argument("attempt", type=positive_int) + reminder_parser.add_argument("agent_id") + sub.add_parser("build-score-packets") + scorer_prompt = sub.add_parser("scorer-prompt") + scorer_prompt.add_argument("mode", choices=MODES) + scorer_prompt.add_argument("scorer", choices=SCORERS) + scorer_prompt.add_argument("attempt", type=positive_int) + scorer_prompt.add_argument("output", type=Path) + record_score_parser = sub.add_parser("record-score") + record_score_parser.add_argument("mode", choices=MODES) + record_score_parser.add_argument("scorer", choices=SCORERS) + record_score_parser.add_argument("attempt", type=positive_int) + record_score_parser.add_argument("output", type=Path) + record_score_parser.add_argument("agent_id") + invalid_score_parser = sub.add_parser("record-invalid-score") + invalid_score_parser.add_argument("mode", choices=MODES) + invalid_score_parser.add_argument("scorer", choices=SCORERS) + invalid_score_parser.add_argument("attempt", type=positive_int) + invalid_score_parser.add_argument("output", type=Path) + invalid_score_parser.add_argument("agent_id") + invalid_score_parser.add_argument("evidence") + disagreements_parser = sub.add_parser("build-disagreements") + disagreements_parser.add_argument("mode", choices=MODES) + adjudication_packet = sub.add_parser("build-adjudication-packet") + adjudication_packet.add_argument("mode", choices=MODES) + adjudicator_prompt = sub.add_parser("adjudicator-prompt") + adjudicator_prompt.add_argument("mode", choices=MODES) + adjudicator_prompt.add_argument("attempt", type=positive_int) + adjudicator_prompt.add_argument("output", type=Path) + record_adjudication_parser = sub.add_parser("record-adjudication") + record_adjudication_parser.add_argument("mode", choices=MODES) + record_adjudication_parser.add_argument("attempt", type=positive_int) + record_adjudication_parser.add_argument("output", type=Path) + record_adjudication_parser.add_argument("agent_id") + invalid_adjudication_parser = sub.add_parser("record-invalid-adjudication") + invalid_adjudication_parser.add_argument("mode", choices=MODES) + invalid_adjudication_parser.add_argument("attempt", type=positive_int) + invalid_adjudication_parser.add_argument("output", type=Path) + invalid_adjudication_parser.add_argument("agent_id") + invalid_adjudication_parser.add_argument("evidence") + evaluator_start_parser = sub.add_parser("evaluator-start") + evaluator_start_parser.add_argument("kind", choices=("scorer", "adjudicator")) + evaluator_start_parser.add_argument("identity") + evaluator_start_parser.add_argument("attempt", type=positive_int) + evaluator_start_parser.add_argument("agent_id") + evaluator_prelaunch = sub.add_parser("record-evaluator-prelaunch-failure") + evaluator_prelaunch.add_argument("kind", choices=("scorer", "adjudicator")) + evaluator_prelaunch.add_argument("identity") + evaluator_prelaunch.add_argument("attempt", type=positive_int) + evaluator_prelaunch.add_argument("output", type=Path) + evaluator_prelaunch.add_argument("evidence") + failed_evaluator = sub.add_parser("record-failed-evaluator") + failed_evaluator.add_argument("kind", choices=("scorer", "adjudicator")) + failed_evaluator.add_argument("identity") + failed_evaluator.add_argument("attempt", type=positive_int) + failed_evaluator.add_argument("output", type=Path) + failed_evaluator.add_argument("agent_id") + failed_evaluator.add_argument("disposition", choices=sorted(INFRA_FAILURE_CODES)) + failed_evaluator.add_argument("evidence") + merge_parser = sub.add_parser("merge-final") + merge_parser.add_argument("mode", choices=MODES) + sub.add_parser("aggregate") + args = parser.parse_args() + + operational_commands = { + "record-freeze-lock", + "prepare-cell", + "report-prompt", + "agent-start", + "record-prelaunch-failure", + "reminder-text", + "record-report", + "record-failed-report", + "build-score-packets", + "scorer-prompt", + "record-score", + "record-invalid-score", + "build-disagreements", + "build-adjudication-packet", + "adjudicator-prompt", + "record-adjudication", + "record-invalid-adjudication", + "evaluator-start", + "record-evaluator-prelaunch-failure", + "record-failed-evaluator", + "merge-final", + "aggregate", + } + operation_lock_handle = None + if args.command in operational_commands: + operation_lock_handle = acquire_operation_lock() + terminal_invalid_event = any( + event["event"] in {"invalid_output_preserved", "run_invalidated"} + for event in event_records() + ) + invalid_marker = RUN / "INVALID.json" + if invalid_marker.exists() or invalid_marker.is_symlink() or terminal_invalid_event: + raise SystemExit("run is INVALID; no further evaluation command is permitted") + validate_static(require_lock=True, announce=False) + if args.command == "record-freeze-lock": + if any(event["event"] == "freeze_locked" for event in event_records()): + raise SystemExit("freeze lock was already recorded") + else: + assert_freeze_locked() + + if args.command == "verify-static": + validate_static(args.locked) + elif args.command == "self-test": + self_test() + elif args.command == "write-file-manifest": + if FILE_MANIFEST.exists(): + raise FileExistsError(FILE_MANIFEST) + write_once(FILE_MANIFEST, render_file_manifest()) + print(f"{FILE_MANIFEST.relative_to(RUN)} {sha256_file(FILE_MANIFEST)}") + elif args.command == "record-freeze-lock": + record_freeze_lock() + elif args.command == "prepare-cell": + prepare_cell(args.run_id, args.runtime) + elif args.command == "report-prompt": + verify_runtime(args.run_id, args.runtime) + print(render_report_prompt(args.run_id, args.runtime), end="") + elif args.command == "record-report": + record_report( + args.run_id, + args.attempt, + args.runtime, + args.agent_id, + args.scope_deviation, + args.scope_evidence, + ) + elif args.command == "record-failed-report": + preserve_failed_report_attempt( + args.run_id, + args.attempt, + args.runtime, + args.agent_id, + args.disposition, + args.evidence, + args.scope_deviation, + ) + elif args.command == "agent-start": + record_agent_start(args.run_id, args.attempt, args.agent_id) + elif args.command == "record-prelaunch-failure": + record_prelaunch_failure(args.run_id, args.evidence) + elif args.command == "reminder-text": + record_reminder(args.run_id, args.attempt, args.agent_id) + print(report_reminder_text(), end="") + elif args.command == "build-score-packets": + build_score_packets() + elif args.command == "scorer-prompt": + source_packet = SCORING / "packets" / args.mode / args.scorer + verify_score_packet(args.mode, args.scorer) + packet = prepare_evaluator_runtime( + "scorer", + f"{args.mode}-{args.scorer}", + args.attempt, + source_packet, + args.output, + ) + print( + render_packet_prompt("scorer.md", packet, args.output, SCORER_ID=args.scorer), + end="", + ) + elif args.command == "record-score": + record_score(args.mode, args.scorer, args.attempt, args.output, args.agent_id) + elif args.command == "record-invalid-score": + record_invalid_evaluator( + "scoring", + f"{args.mode}-{args.scorer}", + args.attempt, + args.output, + args.agent_id, + args.evidence, + ) + elif args.command == "build-disagreements": + build_disagreements(args.mode) + elif args.command == "build-adjudication-packet": + build_adjudication_packet(args.mode) + elif args.command == "adjudicator-prompt": + source_packet = SCORING / "adjudication-packets" / args.mode + verify_adjudication_packet(args.mode) + packet = prepare_evaluator_runtime( + "adjudicator", args.mode, args.attempt, source_packet, args.output + ) + print(render_packet_prompt("adjudicator.md", packet, args.output), end="") + elif args.command == "record-adjudication": + record_adjudication(args.mode, args.attempt, args.output, args.agent_id) + elif args.command == "record-invalid-adjudication": + record_invalid_evaluator( + "adjudication", + args.mode, + args.attempt, + args.output, + args.agent_id, + args.evidence, + ) + elif args.command == "evaluator-start": + record_evaluator_start(args.kind, args.identity, args.attempt, args.agent_id) + elif args.command == "record-evaluator-prelaunch-failure": + record_evaluator_prelaunch_failure( + args.kind, args.identity, args.attempt, args.output, args.evidence + ) + elif args.command == "record-failed-evaluator": + preserve_failed_evaluator_attempt( + "scoring" if args.kind == "scorer" else "adjudication", + args.identity, + args.attempt, + args.output, + args.agent_id, + args.disposition, + args.evidence, + ) + elif args.command == "merge-final": + merge_final(args.mode) + elif args.command == "aggregate": + aggregate() + + +if __name__ == "__main__": + main() diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/blind-map.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/blind-map.tsv new file mode 100644 index 0000000000..570f3aabda --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/blind-map.tsv @@ -0,0 +1,81 @@ +mode label run_id +S A r035 +S B r075 +S C r027 +S D r025 +S E r059 +S F r014 +S G r074 +S H r002 +S I r054 +S J r036 +C A r030 +C B r004 +C C r046 +C D r079 +C E r077 +C F r023 +C G r010 +C H r043 +C I r051 +C J r062 +X A r066 +X B r006 +X C r018 +X D r015 +X E r022 +X F r042 +X G r049 +X H r034 +X I r078 +X J r064 +Q A r008 +Q B r068 +Q C r060 +Q D r026 +Q E r048 +Q F r056 +Q G r031 +Q H r045 +Q I r001 +Q J r065 +W A r070 +W B r032 +W C r024 +W D r041 +W E r050 +W F r053 +W G r009 +W H r012 +W I r044 +W J r076 +M A r013 +M B r067 +M C r071 +M D r047 +M E r021 +M F r003 +M G r063 +M H r037 +M I r057 +M J r019 +R A r069 +R B r038 +R C r028 +R D r011 +R E r052 +R F r007 +R G r040 +R H r072 +R I r055 +R J r029 +K A r020 +K B r016 +K C r005 +K D r039 +K E r073 +K F r033 +K G r017 +K H r058 +K I r080 +K J r061 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/condition-map.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/condition-map.tsv new file mode 100644 index 0000000000..5afacde848 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/condition-map.tsv @@ -0,0 +1,3 @@ +condition_label role package_path tree_sha256 skill_sha256 +c0 v2 frozen-packages/40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897 40b4171cc9daf7e51ba032aef52157a85a49c4c12cea8696deadb948e0867897 a0a75ef8a14497aa78b50b459981097ee99605c57fec95c637cf59aaa20fe766 +c1 v3 frozen-packages/668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf 668f70202c7bc8f23f7f894fb784a9629fd292c7f6fe69ede815b0e4c10137bf 0e23f7747cc63014bade7543efaf745e7e9a7e5d6dee2a48c602ef7a3eba091e diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/launch-schedule.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/launch-schedule.tsv new file mode 100644 index 0000000000..57b78f4de9 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/launch-schedule.tsv @@ -0,0 +1,81 @@ +run_id cell_id wave target_label condition_label replicate +r001 0fea6fbdfedb1e8f9c1c331f5a47fa16 1 m7 c0 1 +r002 4886a308ee81e3c948ea133df9661771 1 m3 c1 1 +r003 fd82077e8d40fcaf1f5a51da67a9bc70 1 m0 c0 1 +r004 a01a89ac57f92484e62f719916fabcde 1 m6 c0 1 +r005 c51d935dd66af61d6ae7370471dd270b 1 m1 c0 1 +r006 237314e935a671dda9f0b729a9164e42 1 m4 c0 1 +r007 2d557eebd147400908963542447f78d8 1 m2 c1 1 +r008 b7d49044025495a68243c59ea223d392 1 m7 c1 1 +r009 4a591e9e7f2b1706efec55c9b809b15e 1 m5 c1 1 +r010 b4c7cb4a6c9a884747af60df5bbe135e 1 m6 c1 1 +r011 df3cf01648593724319ee30d40e53694 1 m2 c0 1 +r012 b9449281466536eb625d875c9a1b52c6 1 m5 c0 1 +r013 c08c9cbec10b09416dedb07222a8fa89 1 m0 c1 1 +r014 225c7b447026b9d4fb60d0953329f336 1 m3 c0 1 +r015 4dfb61f35f3ba9347c43f9c7390941ba 1 m4 c1 1 +r016 0ddb17b601260a6fa49fd03e146c8bf4 1 m1 c1 1 +r017 c0867a2b7f8ca30ce79eaf94a8c2813b 2 m1 c0 3 +r018 ef1a0677a6ed30a4b2ccf4bf2c4fa075 2 m4 c1 3 +r019 07041e4c7851c15fbd8ca3b5b89a649c 2 m0 c1 3 +r020 0a93b4a0423042598341734b5b80aea2 2 m1 c1 3 +r021 45d746368e63763aae22c259c646af5e 2 m0 c0 3 +r022 cf391d64b44517b85ca71858bb6d8e52 2 m4 c0 3 +r023 136b2c4efefe0bab1417f14b38a57793 2 m6 c1 3 +r024 aea81024b16085dc0bc078cab31e9b6d 2 m5 c1 3 +r025 83498ad4b1212eaea7b615f0292a1ce3 2 m3 c0 3 +r026 b9a35f06e08e96ce9b0b093da1de80da 2 m7 c0 3 +r027 a29afd5bf972ec132f78543ba750e93f 2 m3 c1 3 +r028 8beb0474c46f57c79e68aa2f12dfd6a6 2 m2 c0 3 +r029 a3f969c74a0d22bd838ad11dd92aa415 2 m2 c1 3 +r030 919322be3592430d06791a08b1aa8cea 2 m6 c0 3 +r031 c42948b54a7eab85fd103b78c835c7ac 2 m7 c1 3 +r032 418a4bcde6dda1a6187bae8547d85a93 2 m5 c0 3 +r033 ca8819a9835763564d3e5c7b365393d8 3 m1 c0 5 +r034 1ff38d28876c4cdd6329bfeaa1385e6f 3 m4 c1 5 +r035 07fde893dca23f5e7b0b0e576c212375 3 m3 c0 5 +r036 bebc72c4981368d1bffe7e85d5e68357 3 m3 c1 5 +r037 eef95e01264a728a7c6eabe725507b5a 3 m0 c0 5 +r038 628efb38e96428481f2598bd5207472d 3 m2 c1 5 +r039 ba7abe98e80c53f75aa8f8238c7ffe03 3 m1 c1 5 +r040 de3c340e44bf752186e73da5cdbc05bd 3 m2 c0 5 +r041 1ec17f7769c0078f9fcbed12eb0be0f7 3 m5 c0 5 +r042 5edc22ffe7b0779af2a68e40283db476 3 m4 c0 5 +r043 9fed5500b2156c7c3cb16aff89a5a89f 3 m6 c0 5 +r044 32e9f759f85e81477aeb318b0e0497c2 3 m5 c1 5 +r045 434b916b4f0630b6e6aa447c4048f8d6 3 m7 c0 5 +r046 8a37e9f35d221302948bb0526f432b98 3 m6 c1 5 +r047 86e15f99caa5c1efc10a11768832390e 3 m0 c1 5 +r048 aeea1b0c5420db0d4c06d58b2708366e 3 m7 c1 5 +r049 16cb821ea568942a5847dc01ce421b41 4 m4 c0 2 +r050 ada6c44a0f72bc924d6b9f094cad0ff0 4 m5 c0 2 +r051 d1bfa4935ce8286684864898b004f962 4 m6 c0 2 +r052 f0309a354f74b08a1a0972b62fac6ccd 4 m2 c1 2 +r053 296be450c238efd67a62d38623894010 4 m5 c1 2 +r054 7da22994e57951eb951e0e387d852650 4 m3 c1 2 +r055 4612a4154bf19ccc48a84d507fe9b05b 4 m2 c0 2 +r056 e6632dafee22c0cfb734d7911d78769f 4 m7 c0 2 +r057 1ab98c2830b91afc592e74e6e6c5b9be 4 m0 c1 2 +r058 bf0abeb5157d3eed578e8bc0edde0d47 4 m1 c0 2 +r059 91104a670ad522d0fe606ac8e15dc1f5 4 m3 c0 2 +r060 02038086e3c763683e357ad7d23a8005 4 m7 c1 2 +r061 b0f33b6db364346c2523b34bcc8a0ecd 4 m1 c1 2 +r062 359c5fe574e54bea738b01822c7104f6 4 m6 c1 2 +r063 812bd420e5eba308dc68e07047e583c1 4 m0 c0 2 +r064 5f63b439ef17bd4c9938d31e4445e847 4 m4 c1 2 +r065 2505caff3f06d70966adcc385f251627 5 m7 c1 4 +r066 da357b62c59f84a5e9f0f0a11214900a 5 m4 c1 4 +r067 ae051f5093ea311894002930e26944f6 5 m0 c1 4 +r068 6a4a5f007b65de786b2391d09453d5e7 5 m7 c0 4 +r069 40e593bb6d93cf7b49e1fb35342ff99a 5 m2 c0 4 +r070 1769a58b00a1db740e39dbd7cba09c6e 5 m5 c0 4 +r071 4cec2104444a8240d06ee311657dc0f3 5 m0 c0 4 +r072 843cbdc69d6a93ba2d97c2ef54f74264 5 m2 c1 4 +r073 59206a03a32ed8ca983f8665c26fcacc 5 m1 c0 4 +r074 6367c11d5940c2dbeeb37e422d2b79c8 5 m3 c0 4 +r075 02d88d35c49cd6779edc8578e7d89bdb 5 m3 c1 4 +r076 ecf25044b6f65e28e446370bea636c89 5 m5 c1 4 +r077 886d6d32ed0e40aa1f82f3f94cfcbcaf 5 m6 c1 4 +r078 2a9b5392e246efa52195b90613899322 5 m4 c0 4 +r079 c8dd0a199fe8db9173fe43372c47e05b 5 m6 c0 4 +r080 aaf0db751da7393ff60ded2b6d179fb7 5 m1 c1 4 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/presentation-orders.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/presentation-orders.tsv new file mode 100644 index 0000000000..744fed7552 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/presentation-orders.tsv @@ -0,0 +1,17 @@ +claim labels_in_order +S-s1 A,B,E,F,G,I,J,C,D,H +S-s2 J,E,F,I,B,G,H,C,D,A +C-s1 H,F,J,C,B,A,I,G,E,D +C-s2 I,J,G,H,F,D,A,C,E,B +X-s1 C,F,I,B,A,H,G,D,J,E +X-s2 H,D,B,C,F,E,A,J,G,I +Q-s1 D,B,E,F,A,I,G,J,H,C +Q-s2 C,B,D,A,G,J,I,F,H,E +W-s1 H,F,B,E,C,A,I,D,J,G +W-s2 F,G,E,C,A,B,H,J,D,I +M-s1 F,E,D,G,A,C,J,I,H,B +M-s2 G,J,A,B,F,E,H,C,I,D +R-s1 H,F,G,I,A,D,J,E,C,B +R-s2 A,J,G,B,F,I,C,E,D,H +K-s1 H,D,J,I,F,G,A,C,B,E +K-s2 J,F,H,I,A,B,E,D,C,G diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/scoring-schedule.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/scoring-schedule.tsv new file mode 100644 index 0000000000..40a5798be0 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/scoring-schedule.tsv @@ -0,0 +1,17 @@ +claim +S-s1 +M-s1 +Q-s2 +M-s2 +W-s2 +R-s2 +W-s1 +Q-s1 +S-s2 +X-s1 +X-s2 +R-s1 +C-s1 +C-s2 +K-s1 +K-s2 diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/seeds.json b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/seeds.json new file mode 100644 index 0000000000..173d339364 --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/seeds.json @@ -0,0 +1,7 @@ +{ + "condition": "cd806c526988b02b7ec4c4cb13a20f4bddb2e175787e20044c65c87e7dba5b8e", + "schedule": "bacd8b99a28e6368f7d0fb8996bf5fff0546407c552e6315135ea6509ef4ac79", + "blind": "270254466233a651d3e442a7a5394cf40db13d824b84cd702a75501960815522", + "presentation": "893456b29122ef6e27acb3a82f3f5c08cb266c1a7b246a98eaaa072f6de1ee0a", + "scorer": "ea4a345be348a08359dbd6994b92b83cc3241d714fa165a5f8254462372ad755" +} diff --git a/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/target-map.tsv b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/target-map.tsv new file mode 100644 index 0000000000..7b6e4fbfbb --- /dev/null +++ b/evals/unsafe-rust/runs/2026-07-31-v3-targeted/sealed/target-map.tsv @@ -0,0 +1,9 @@ +target_label mode source_path tree_sha256 word_cap +m0 M fixtures/v3-targeted/m_multirelease b269cf068196d1c06b87be6bcded494827e7ac8a2cb6debf1eb0f0f5d0388479 1800 +m1 K fixtures/v3-targeted/k_regression ca272e524b36a892e25f6169631a184ff764026451eff2f6ac4ab8e9d5e87ea2 2200 +m2 R fixtures/v3-targeted/r_redesign 6d1a41909b012484d71f94d194071a7ebbb6773b7596db88127ca2a195b70ffc 1800 +m3 S fixtures/v3-targeted/s_symbolic 28ecc523e15b914a187814ab2752c0d85996948a552c62f970d8484bc6ed467a 1800 +m4 X fixtures/v3-targeted/x_cross 25b4efef689601f3b5983bf6b914bd367153d0b21a6f1f4d40de50c5d412afa7 2400 +m5 W fixtures/v3-targeted/w_whole_execution b27b95fbc9ffa9d335bb6b4614a9f227a5798122fca79b42cf53c9b106a6aff6 1800 +m6 C fixtures/v3-targeted/c_conflict 065c3cfc032af93e7576e17e49322826c4a379a707870175b16c2663d1e8e4e0 1800 +m7 Q fixtures/v3-targeted/q_quantifiers c0a4c43373a159cb38d08724af8b02187b249ab6a73f7e1b10b2276f38b0cb5a 1800