diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 90c644c8..2260e283 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -661,6 +661,8 @@ def checked_board(): problems.append(f"no-tools {model}: {error}") spent = exact_spend(costs) try: + if type(design.get("spent_before")) not in (int, float): + raise ValueError("spent_before must be a nonboolean number") declared = float(design.get("spent_before")) except (TypeError, ValueError, OverflowError): declared = math.nan @@ -989,7 +991,8 @@ def no_tool(arguments): cls, _ = classify(record, 0.0, 0.0, None) outcome = action(cls, n) self.say( - f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record):.6f}" + f"{model} {sid} {n} {cls} {outcome} " + f"${attempt_cost(record, saved=False):.6f}" ) if self.interrupted.is_set(): self.states[model] = "interrupted" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index d0957ef7..c20085b8 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1,4 +1,4 @@ -"""The confirmatory calculator-interface run. +"""The confirmatory calculator-interface and web-search stages. Usage: python scripts/confirmatory_run.py run DESIGN [--workers N] [--dry-run] @@ -7,13 +7,20 @@ Runs the frozen pre-registration's "Run procedure" and writes its "Run records", which the confirmatory report checks before it scores anything. DESIGN is the JSON file written at the freeze, the file the report reads. +Its required ``kind`` selects the registered calculator or web stage. The +calculator stage keeps the six models and their original arm labels. The web +stage compares ``web`` and ``web+pe3`` on Luna, Flash-Lite and Sonnet, after +the hashed calculator design's stage has closed. Its frozen inputs and scoring +metadata agree with that design, and its prior spend includes both earlier +stages. Both web profiles prohibit benchmark searches, as in development. - **Lanes.** Each model has two lanes. Households alternate between them in draw order, the first drawn to lane 1. A lane runs one household's arms one after another, in the order drawn for that household and model (the stream - ``order/``), then its reruns in passes (second attempts before + ``order/`` for calculator, ``order/web/`` for web), then its + reruns in passes (second attempts before third, each pass in the household's order), and only then decides its next - household. The six models run at once. + household. A stage's models run at once: six calculator models or three web models. - **One engine pool per model**, started once, with no features of its own. Each call carries its arm's features in a call frame (``tool_call_frame``), the message a pool started with those features would send. ``setup-start`` @@ -28,8 +35,9 @@ households has ended a pass whose reruns are undecided, so that whether it has finished is known. - **Spend.** No conversation is stopped for spend, except at the hard - ceiling: the overall cap plus $25, over the spend before the calculator - stage and every model's spend since. At the crossing, no attempt starts, + ceiling: the overall cap plus $25, over the spend before the selected + stage and every model's spend since. The web design's whole-run cap is $270. + At the crossing, no attempt starts, the conversations under way end ``budget_stop`` at their next turn, and every model's run log gets a ``ceiling`` event at the crossing's time. The decision to admit a household and the crossing share one lock, so no @@ -93,7 +101,11 @@ treatment commit's (c0a2885b), its tree is clean at the design's run commit, the machine is on AC power with the lid open, and the runs folder holds no records: a run is never resumed. ``--workers`` must equal the design's -engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the +engine workers per model. A Gemini web model also requires the exact LiteLLM +version checked by the treatment's search patch at preflight. During execute, +after setup-start and before the completion factory or any completion call, +the runner applies that patch so search survives beside function tools. +An interrupt (SIGINT or SIGTERM) only notes the signal; the main loop then stops every model's lanes with cause ``interrupt`` and lets the conversations under way end. A second interrupt exits at once until cleanup ignores signals before its final check. Signals @@ -105,6 +117,7 @@ import argparse import hashlib +import importlib.metadata import json import math import os @@ -132,6 +145,11 @@ import tool_pilot_argon as pilot # noqa: E402 TREATMENT_COMMIT = "c0a2885bfc8f5df59a93458230c445befb986dcb" +ANCHOR_REPOSITORY = "PolicyEngine/policybench-v2-anchors" +ANCHOR_REFS = { + "calculator": "refs/heads/anchors/v2-confirmatory", + "web": "refs/heads/anchors/v2-confirmatory-web", +} # The treatment's code, which must be the treatment commit's. TREATMENT_PATHS = ( "scripts/tool_pilot_argon.py", @@ -151,6 +169,99 @@ "loo-labels-flat", ) FRONTIER_ARMS = ("pe", "pe3") +WEB_MODELS = ("gpt-6-luna", "gemini-3.5-flash-lite", "claude-sonnet-5.5") +WEB_ARMS = ("web", "web+pe3") + + +@dataclass(frozen=True) +class Stage: + """A registered stage's models, labels, folders, stream and profile rule.""" + + kind: str + models: tuple[str, ...] + model_arms: dict[str, tuple[str, ...]] + folder_keys: dict[str, str] + order_stream: str + profile_rule: str + calculator_label: str + fallback_arms: tuple[str, ...] | None = None + fallback_folder_key: str | None = None + + def arms_for(self, model: str) -> tuple[str, ...]: + """Keep calculator-only explicit model overrides on the frontier arms.""" + if model in self.model_arms: + return self.model_arms[model] + if self.fallback_arms is not None: + return self.fallback_arms + raise KeyError(f"{model}: no model in the {self.kind} stage") + + def folder_key_for(self, model: str) -> str: + """The model's registered mapping, or the calculator override mapping.""" + if model in self.folder_keys: + return self.folder_keys[model] + if self.fallback_folder_key is not None: + return self.fallback_folder_key + raise KeyError(f"{model}: no model in the {self.kind} stage") + + +STAGES = { + "calculator": Stage( + "calculator", + CHEAP + FRONTIER, + {m: ARMS if m in CHEAP else FRONTIER_ARMS for m in CHEAP + FRONTIER}, + {m: "arms" if m in CHEAP else "frontier_arms" for m in CHEAP + FRONTIER}, + "order/{model}", + "calculator", + "pe3", + FRONTIER_ARMS, + "frontier_arms", + ), + "web": Stage( + "web", + WEB_MODELS, + {m: WEB_ARMS for m in WEB_MODELS}, + {m: "arms" for m in WEB_MODELS}, + "order/web/{model}", + "web", + "web+pe3", + ), +} + +# The web stage uses precisely the calculator stage's frozen inputs and settings. +SHARED_STAGE_FIELDS = ( + "preregistration", + "frozen_sha256", + "board", + "engine", + "eligible_pool", + "draw_order", + "treatment_harness", + "run_commit", + "workers", + "no_tools", + "development_runs", +) +WEB_SCORING_FIELDS = ( + "build", + "exclusions_sha256", + "baseline_exclusions", + "supplementary_exclusions", + "disputed", + "adjudication", + "adjudicator_model", + "judge", + "published_sources", +) + + +def stage_for(design: dict) -> Stage: + """Refuse a missing or unregistered kind before any stage starts.""" + kind = design.get("kind") + if not isinstance(kind, str) or kind not in STAGES: + raise ValueError("the design's kind must be calculator or web") + return STAGES[kind] + + # The features each label drops from pe3; None for the tool v1 arm. DROPPED = { "pe": None, @@ -168,6 +279,12 @@ HOST_LOG_GAP = 90.0 # seconds SAMPLE_SECONDS = 30.0 CEILING_MARGIN = 25.0 # dollars above the overall cap +CALCULATOR_OVERALL_CAP = 150.0 +CALCULATOR_CAP = 145.0 +WEB_OVERALL_CAP = 270.0 +WEB_CAP = 120.0 +CAP_TOLERANCE = 0.005 # dollars: a cap rounded to the cent +RESERVE_TOLERANCE = 0.0005 # dollars: a reserve rounded to a tenth of a cent ADMISSION_SLACK = 1e-9 # dollars EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the run's start WAIT_SECONDS = 1.0 # how often a waiting lane looks again @@ -237,16 +354,17 @@ EARLY = frozenset({"operator", "harness", "ceiling", "none"}) REQUIRED = ( + "kind", "preregistration", "frozen_sha256", "runs", + "development_runs", "stage", "board", "engine", "eligible_pool", "draw_order", "arms", - "frontier_arms", "treatment_harness", "run_commit", "caps", @@ -281,11 +399,16 @@ def draw(master: int, pool_ids: list[str], n: int | None = None) -> list[str]: def orders( - master: int, model: str, draw_order: list[str], arms: tuple[str, ...] + master: int, + model: str, + draw_order: list[str], + arms: tuple[str, ...], + *, + kind: str = "calculator", ) -> dict[str, list[str]]: """Each drawn household's arm order for a model, drawn for every household in draw order, so no order depends on which households run.""" - generator = stream(master, f"order/{model}") + generator = stream(master, STAGES[kind].order_stream.format(model=model)) return { sid: [str(a) for a in generator.permutation(list(arms))] for sid in draw_order } @@ -296,6 +419,8 @@ def orders( def arm_string(label: str) -> str: """The harness's arm string for a design label (`TOOL_FEATURES` order).""" + if label in WEB_ARMS: + return label dropped = DROPPED[label] if dropped is None: return "pe" @@ -305,8 +430,36 @@ def arm_string(label: str) -> str: def arm_profile(model: str, label: str) -> pilot.ModelProfile: - """The treatment harness's profile for a model and design label; refused - unless its calculator features are the ones the label registers.""" + """The pinned profile, checked by its stage's registered treatment rule. + + Web profiles retain development's benchmark-search prohibition: the pinned + harness applies it to both web arms and includes it in their notices. + """ + stage = STAGES["web" if label in WEB_ARMS else "calculator"] + if stage.profile_rule == "web": + profile = pilot.model_profile( + model, + arm=label, + prompt_version=PROMPT_VERSION, + benchmark_search_rule=True, + ) + if not profile.uses_web: + raise ValueError(f"{model}/{label}: the harness gives no web search") + if label == "web" and profile.uses_calculator: + raise ValueError(f"{model}/{label}: the harness gives a calculator") + if label == "web+pe3" and ( + not profile.uses_calculator + or set(profile.tool_features) != set(pilot.TOOL_FEATURES) + ): + raise ValueError( + f"{model}/{label}: the harness lacks the full calculator features" + ) + if not profile.benchmark_search_rule: + raise ValueError( + f"{model}/{label}: the harness omits the benchmark-search rule" + ) + pilot.web_search_tool(profile) + return profile profile = pilot.model_profile( model, arm=arm_string(label), prompt_version=PROMPT_VERSION ) @@ -427,8 +580,28 @@ def action(cls: str, attempt: int) -> str: return "stands" -def attempt_cost(record: dict) -> float: - return float((record.get("totals") or {}).get("cost_usd") or 0.0) +def attempt_cost(record: dict, *, saved: bool = True) -> float: + """A saved attempt must account explicitly for its whole recorded cost. + + Live accounting keeps its existing behavior: the harness writes the total, + including error/deadline stubs, and malformed output still gets recorded. + Validation of evidence read back from disk must never invent a zero cost. + """ + if not saved: + return float((record.get("totals") or {}).get("cost_usd") or 0.0) + totals = record.get("totals") + if not isinstance(totals, dict): + raise ValueError("attempt totals must be an object with cost_usd") + value = totals.get("cost_usd") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("attempt cost_usd must be a present number") + try: + cost = float(value) + except OverflowError as error: + raise ValueError("attempt cost_usd must be finite and nonnegative") from error + if not math.isfinite(cost) or cost < 0: + raise ValueError("attempt cost_usd must be finite and nonnegative") + return cost _clock_lock = threading.Lock() @@ -436,13 +609,18 @@ def attempt_cost(record: dict) -> float: def clock() -> float: - """`time.time()`, never earlier than a time it gave before. The wall clock - can step back by about 0.1 ms, even within a thread, and the run records order - attempts, stops, admission decisions and the ceiling's crossing by the - times this gives; each is taken under the lock that orders it.""" + """Return a time strictly greater than every earlier runner reading. + + When the wall clock repeats or steps back, advance by one representable + float. The lock orders attempts, stops, admission decisions and the + ceiling's crossing across all threads. + """ global _clock_last with _clock_lock: - _clock_last = max(time.time(), _clock_last) + now = time.time() + _clock_last = ( + now if now > _clock_last else math.nextafter(_clock_last, math.inf) + ) return _clock_last @@ -816,7 +994,7 @@ class Ceiling(pilot.Spend): def __init__(self, ceiling_usd: float, spent_before: float): super().__init__(ceiling_usd, spent_before) - self.crossed_at: float | None = None + self.crossed_at: float | None = clock() if spent_before >= ceiling_usd else None self.closed = False # the records are final: spend is no longer counted self.unrecorded_usd = 0.0 # spend after that, in no record @@ -973,7 +1151,9 @@ def __init__(self, run: "Run", model: str, labels: tuple[str, ...], folders: dic self.profiles = {label: arm_profile(model, label) for label in labels} self.cap = float(design["caps"][model]) self.reserve = float(design["reserves"][model]) - self.orders = orders(run.master, model, run.draw_order, labels) + self.orders = orders( + run.master, model, run.draw_order, labels, kind=run.stage.kind + ) self.cond = threading.Condition() self._log_lock = threading.Lock() self.state: dict[str, str] = {} # household -> running, settling, finished @@ -1155,7 +1335,7 @@ def end_attempt( abandoned at its deadline.""" with self.cond: end = clock() - self.spent[sid] += attempt_cost(record) + self.spent[sid] += attempt_cost(record, saved=False) attempt = Attempt(label, n, start, end, record) if retained is not None: retained.append(attempt) # recovery owns it before any event I/O @@ -1230,6 +1410,8 @@ def converse(self, sid: str, label: str, n: int) -> tuple[dict, bool]: spend = AttemptSpend(self.run.ceiling) def tool_call(arguments): + if not profile.uses_calculator: + raise RuntimeError(f"{self.model}/{label}: no calculator is offered") message = ( pilot.tool_call_frame(features, arguments) if features else arguments ) @@ -1394,7 +1576,7 @@ def passes( write_json(in_flight / f"{label}-{n}.json", record) self.run.say( f"{self.model} lane {lane} {sid} {label} #{n}: " - f"{record.get('status')} ${attempt_cost(record):.4f} " + f"{record.get('status')} ${attempt_cost(record, saved=False):.4f} " f"(total ${self.run.ceiling.spent_usd:.2f})" ) if not done: @@ -1490,17 +1672,23 @@ def lane(self, lane: int) -> None: def manifest(self) -> dict: board, design = self.run.board, self.run.design - pe3 = arm_profile(self.model, "pe3") + pe3 = self.profiles[self.run.stage.calculator_label] first = sorted(board["scenarios"])[0] def timeout(profile): built = pilot.build_initial_request( board["scenarios"][first], board["variables"][first], profile ) - return built[1].get("timeout") + value = built[1].get("timeout") + return ( + max(float(value or 0), pilot.WEB_REQUEST_TIMEOUT_SECONDS) + if profile.uses_web + else value + ) board_path = Path(design["board"]) return { + "design_sha256": self.run.design_sha256, "started_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "git_head": pilot._git("rev-parse", "HEAD"), "git_dirty": bool(pilot._git("status", "--porcelain")), @@ -1538,6 +1726,18 @@ def timeout(profile): **pilot.profile_treatment(profile), "tool_version": profile.tool_version, "request_timeout_seconds": timeout(profile), + **( + { + "web_search_tool": pilot.web_search_tool(profile), + "benchmark_search_rule": profile.benchmark_search_rule, + "web_timeouts_retried": False, + "search_usd_per_1k": pilot.SEARCH_USD_PER_1K[ + profile.provider + ], + } + if self.run.stage.kind == "web" + else {} + ), } for label, profile in self.profiles.items() }, @@ -1613,7 +1813,7 @@ def default_completion(profile: pilot.ModelProfile): class Run: - """The six models' lanes, the host log and the hard ceiling.""" + """The chosen stage's models and lanes, host log and whole-run ceiling.""" def __init__( self, @@ -1625,8 +1825,12 @@ def __init__( completion_for: Callable = default_completion, host_factory: Callable | None = None, out=sys.stdout, + *, + design_sha256: str, ): self.design, self.out = design, out + self.design_sha256 = design_sha256 + self.stage = stage_for(design) self.workers = int(design["workers"]) if workers is None else workers self.converse, self.pool_factory = converse, pool_factory self.completion_for = completion_for @@ -1642,13 +1846,13 @@ def __init__( self.ceiling = Ceiling( float(design["overall_cap"]) + CEILING_MARGIN, float(design["spent_before"]) ) - self.models = tuple(models or CHEAP + FRONTIER) + self.models = tuple(models or self.stage.models) self.lanes = { model: Lanes( self, model, - ARMS if model in CHEAP else FRONTIER_ARMS, - design["arms"] if model in CHEAP else design["frontier_arms"], + self.stage.arms_for(model), + design[self.stage.folder_key_for(model)], ) for model in self.models } @@ -1928,7 +2132,10 @@ def execute(self) -> None: setup_open.add(lanes.model) for model, lanes in self.lanes.items(): self.watch(anchor=False) - lanes.completion = self.completion_for(lanes.profiles["pe3"]) + representative = lanes.profiles[self.stage.calculator_label] + if representative.uses_web and representative.provider == "gemini": + pilot.allow_gemini_search_with_functions() + lanes.completion = self.completion_for(representative) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) lanes.manifest_sha256 = write_json( lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 @@ -2163,12 +2370,714 @@ def host_power() -> tuple[bool, bool]: return "'AC Power'" in batt, '"AppleClamshellState" = No' in lid -def preflight(design: dict, models: tuple[str, ...]) -> list[str]: +def calculator_evidence(design: dict) -> list[float]: + """Require a closed calculator stage and return every recorded cost. + + The existing runner's end evidence is a closed host span, a summary for + every model, empty in-flight inventories, complete attempt records, and + the final anchor covering all current logs and manifests with a successful + final push. Unresolved requests and unsettled attempts prevent closure. + A stopped stage may be closed; its confirmatory standing is a report rule. + """ + from confirmatory_no_tools import ( + read_regular, + saved_children, + saved_json_paths, + ) + + root, frozen = Path(design["runs"]), design["frozen_sha256"] + + def document(path): + value = json.loads(read_regular(path).decode("utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path}: expected a JSON object") + return value + + def attempt_record(path): + # Calculator archives preserve the harness's original bytes, whose + # first attempt need not carry an attempts field. The final adds its + # complete links; the no-tools reader's stronger schema is different. + data = read_regular(path) + record = json.loads(data.decode("utf-8")) + if not isinstance(record, dict) or "prior_attempts" in record: + raise ValueError(f"{path}: invalid calculator attempt record") + if "attempts" in record and not isinstance(record["attempts"], list): + raise ValueError(f"{path}: calculator attempts must be a list") + totals = record.get("totals") + if not isinstance(totals, dict): + raise ValueError(f"{path}: calculator totals must be an object") + cost = totals.get("cost_usd") + if isinstance(cost, bool) or not isinstance(cost, (int, float)): + raise ValueError(f"{path}: calculator cost must be a present number") + if not math.isfinite(float(cost)) or cost < 0: + raise ValueError(f"{path}: calculator cost must be finite and nonnegative") + return record, sha256_bytes(data) + + def lines(path, seed=None): + texts = read_regular(path).decode("utf-8").splitlines() + rows, heads = [], [seed] + for text in texts: + row = json.loads(text) + if not isinstance(row, dict) or ( + seed is not None and row.get("prev") != heads[-1] + ): + raise ValueError(f"{path}: invalid hash chain") + rows.append(row) + heads.append(line_sha(text)) + return rows, heads + + def finite(value): + if ( + isinstance(value, bool) + or not isinstance(value, (float, int)) + or not math.isfinite(value) + ): + raise ValueError("calculator end evidence holds a nonfinite number") + return float(value) + + host = document(root / "host" / "host.json") + if finite(host["end"]) < finite(host["start"]): + raise ValueError("calculator host log has no closed span") + _, host_heads = lines(root / "host" / "host-log.jsonl", HOST_SEED) + anchors_path = root / "anchors.jsonl" + anchors, anchor_heads = lines(anchors_path, chain_seed(frozen, "", "anchors.jsonl")) + if not anchors: + raise ValueError("calculator stage has no final anchor") + costs, heads_by_model = [], {} + for model in STAGES["calculator"].models: + folder = root / model + if saved_children(folder / "in-flight"): + raise ValueError(f"{model}: calculator attempts remain in flight") + summaries = sorted( + p + for p in saved_children(folder) + if p.name.startswith("run-summary-") and p.suffix == ".json" + ) + if not summaries: + raise ValueError(f"{model}: calculator stage has no run summary") + summary = document(summaries[-1]) + if ( + summary.get("model") != model + or type(summary.get("abandoned_running")) is not int + or summary["abandoned_running"] != 0 + ): + raise ValueError(f"{model}: calculator summary leaves a request unresolved") + datetime.strptime(summary["finished_at_utc"], "%Y-%m-%dT%H:%M:%SZ") + run_rows, run_heads = lines( + folder / "run-log.jsonl", chain_seed(frozen, model, "run-log.jsonl") + ) + admissions, admission_heads = lines( + folder / "admission-log.jsonl", + chain_seed(frozen, model, "admission-log.jsonl"), + ) + if any( + row.get("event") == "late-unresolved" or row.get("class") == "unsettled" + for row in run_rows + ): + raise ValueError( + f"{model}: calculator requests are unresolved or unsettled" + ) + if { + row["household"] for row in admissions if row.get("decision") == "admit" + } != set(summary.get("households_finished", [])): + raise ValueError( + f"{model}: calculator admitted households have not finished" + ) + attempts = [row for row in run_rows if "event" not in row] + inventory = {} + model_costs, final_count = [], 0 + for label in STAGES["calculator"].model_arms[model]: + key = STAGES["calculator"].folder_keys[model] + records = folder / design[key][label] / design["stage"] / "households" + for path in saved_json_paths(records): + final_count += 1 + record, digest = attempt_record(path) + links = record["attempts"] + if any( + not isinstance(link, dict) + or type(link.get("n")) is not int + or link["n"] != n + for n, link in enumerate(links, 1) + ): + raise ValueError(f"{path}: calculator archive links are invalid") + inventory[(path.stem, label, len(links) + 1)] = (digest, record) + archives = { + p.name: p + for p in saved_json_paths(records / "attempts") + if p.stem.rsplit("-", 1)[0] == path.stem + } + expected = {f"{path.stem}-{n}.json" for n in range(1, len(links) + 1)} + if set(archives) != expected: + raise ValueError( + f"{path}: calculator archives differ from final links" + ) + for n, link in enumerate(links, 1): + archive = archives[f"{path.stem}-{n}.json"] + prior, prior_digest = attempt_record(archive) + if ( + link.get("file") != f"attempts/{archive.name}" + or link.get("sha256") != prior_digest + ): + raise ValueError( + f"{path}: calculator archive hash or path differs" + ) + inventory[(path.stem, label, n)] = (prior_digest, prior) + # An archive without a final must not hide a request or its spend. + finals = {p.stem for p in saved_json_paths(records)} + for archive in saved_json_paths(records / "attempts"): + if archive.stem.rsplit("-", 1)[0] not in finals: + raise ValueError(f"{archive}: calculator archive has no final") + seen = set() + for row in attempts: + key = (row["household"], row["arm"], row["attempt"]) + if ( + key in seen + or key not in inventory + or inventory[key][0] != row.get("sha256") + ): + raise ValueError( + f"{model}: calculator run log differs from attempt records" + ) + seen.add(key) + model_costs.append(attempt_cost(inventory[key][1])) + if finite(row["end"]) < finite(row["start"]): + raise ValueError(f"{model}: calculator attempt has not ended") + if ( + seen != set(inventory) + or summary.get("attempts") != len(attempts) + or summary.get("final_records") != final_count + ): + raise ValueError(f"{model}: calculator summary or log leaves out attempts") + costs.extend(model_costs) + for row in run_rows: + if row.get("event") == "late-cost": + late = finite(row["cost_usd"]) + if late < 0: + raise ValueError(f"{model}: calculator late spend is negative") + costs.append(late) + manifests = sorted( + p + for p in saved_children(folder) + if p.name.startswith("manifest-") and p.suffix == ".json" + ) + if len(manifests) != 1: + raise ValueError(f"{model}: calculator stage needs exactly one manifest") + heads_by_model[model] = { + "run-log.jsonl": run_heads, + "admission-log.jsonl": admission_heads, + "manifest": sha256_bytes(read_regular(manifests[0])), + } + for i, anchor in enumerate(anchors): + if set(anchor.get("models", {})) != set(heads_by_model): + raise ValueError("calculator anchor does not cover every model") + covered = [(anchor["host-log.jsonl"], host_heads)] + for model, heads in heads_by_model.items(): + if anchor["models"][model].get("manifest") != heads["manifest"]: + raise ValueError(f"{model}: calculator anchor manifest differs") + covered.extend( + (anchor["models"][model][name], heads[name]) + for name in ("run-log.jsonl", "admission-log.jsonl") + ) + for got, heads in covered: + count = got.get("lines") + if ( + type(count) is not int + or not 0 <= count < len(heads) + or got.get("sha256") != heads[count] + ): + raise ValueError("calculator anchor differs from its logs") + if i == len(anchors) - 1 and count != len(heads) - 1: + raise ValueError("calculator final anchor omits log lines") + final_head = anchor_heads[-1] + pushes, _ = lines(root / "anchor-pushes.jsonl") + if ( + not pushes + or pushes[-1].get("anchor") != final_head + or pushes[-1].get("outcome") != "success" + or type(pushes[-1].get("returncode")) is not int + or pushes[-1].get("returncode") != 0 + ): + raise ValueError("calculator final anchor has no successful final push") + if finite(pushes[-1]["ended"]) < finite(pushes[-1]["started"]) or finite( + pushes[-1]["started"] + ) < finite(anchors[-1]["time"]): + raise ValueError("calculator final push has invalid times") + snapshot = root / f"anchors.{final_head}.push-snapshot.jsonl" + if read_regular(snapshot) != read_regular(anchors_path): + raise ValueError("calculator final push snapshot differs from its anchors") + return costs + + +def web_preflight(design: dict) -> list[str]: + """Record malformed web designs as refusals before a provider can run.""" + try: + if not isinstance(design, dict): + raise ValueError("the design must be an object") + return _web_preflight(design) + except ( + OSError, + ValueError, + KeyError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + RuntimeError, + ) as error: + return [f"the web design is malformed: {error}"] + + +def _web_preflight(design: dict) -> list[str]: + """The web stage follows its hashed, closed calculator stage and all spend.""" + from confirmatory_no_tools import ( + component, + exact_spend, + history, + listed_is_dir, + read_regular, + saved_children, + ) + + problems = [] + if design.get("overall_cap") != WEB_OVERALL_CAP: + problems.append("the web design's overall_cap must be 270") + try: + link = design["calculator"] + path = Path(link["path"]) + data = read_regular(path) + if sha256_bytes(data) != link["sha256"]: + return [*problems, "the calculator design does not hash as recorded"] + calculator = json.loads(data.decode("utf-8")) + if not isinstance(calculator, dict) or calculator.get("kind") != "calculator": + return [*problems, "the linked calculator design's kind must be calculator"] + except (OSError, ValueError, KeyError, TypeError, RecursionError) as error: + return [*problems, f"the calculator design cannot be read: {error}"] + if calculator.get("overall_cap") != CALCULATOR_OVERALL_CAP: + problems.append("the linked calculator design's overall_cap must be 150") + problems.extend(f"linked calculator: {p}" for p in rehearsal_problems(calculator)) + problems.extend(f"linked calculator: {p}" for p in stage_start_problems(calculator)) + allocation = allocation_problems(calculator, STAGES["calculator"]) + problems.extend(f"linked calculator: {p}" for p in allocation) + if not allocation: + problems.extend( + f"linked calculator: {p}" + for p in projection_problems(calculator, STAGES["calculator"]) + ) + for field in SHARED_STAGE_FIELDS: + if field not in calculator or design.get(field) != calculator[field]: + problems.append(f"web and calculator designs differ in {field}") + for field in WEB_SCORING_FIELDS: + if design.get(field) != calculator.get(field): + problems.append(f"web and calculator designs differ in {field}") + try: + roots = [ + Path(design["runs"]).resolve(), + Path(design["no_tools"]["runs"]).resolve(), + Path(calculator["runs"]).resolve(), + ] + if any( + roots[0].is_relative_to(other) or other.is_relative_to(roots[0]) + for other in roots[1:] + ): + problems.append( + "the web runs root overlaps the no-tools or calculator runs root" + ) + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: + problems.append(f"the web runs roots are invalid: {error}") + try: + calculator_costs = calculator_evidence(calculator) + costs = list(calculator_costs) + no_tools_costs = [] + spec = design["no_tools"] + for model in saved_children(Path(spec["runs"])): + if listed_is_dir(model): + _, saved = history( + model + / component(spec["folder"]) + / component(spec["stage"]) + / "households" + ) + costs.extend(saved) + no_tools_costs.extend(saved) + if type(calculator["spent_before"]) not in (int, float): + raise ValueError("calculator spent_before must be a nonboolean number") + calculator_spent_before = float(calculator["spent_before"]) + if not math.isfinite(calculator_spent_before) or calculator_spent_before < 0: + raise ValueError("calculator spent_before must be finite and nonnegative") + no_tools_spent = exact_spend(no_tools_costs) + if calculator_spent_before < no_tools_spent - 1e-6: + problems.append( + f"linked calculator spent_before ${calculator_spent_before} is " + f"below no-tools recorded spend ${no_tools_spent}" + ) + spent = max( + exact_spend([calculator_spent_before, *calculator_costs]), + exact_spend(costs), + ) + if type(design["spent_before"]) not in (int, float): + raise ValueError("web spent_before must be a nonboolean number") + declared = float(design["spent_before"]) + if not 0 <= declared < math.inf or declared < spent - 1e-6: + problems.append( + f"web spent_before ${declared} is below no-tools and calculator " + f"recorded spend ${spent}" + ) + except ( + OSError, + ValueError, + KeyError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + ) as error: + problems.append(f"the calculator stage has not ended: {error}") + return problems + + +def rehearsal_problems(design: dict) -> list[str]: + """Rehearsal-only options cannot be present in a frozen design.""" + return [ + f"the confirmatory design cannot contain the rehearsal option {field}" + for field in ("schema_hash_exempt", "stated_engine_overrides") + if field in design + ] + + +def stage_start_problems(design: dict) -> list[str]: + """No provider request may start once the whole-run ceiling is reached.""" + try: + if any( + type(design[field]) not in (int, float) + for field in ("spent_before", "overall_cap") + ): + raise ValueError("stage money must be nonboolean numbers") + spent_before = float(design["spent_before"]) + overall_cap = float(design["overall_cap"]) + ceiling = overall_cap + CEILING_MARGIN + if ( + math.isfinite(overall_cap) + and math.isfinite(spent_before) + and 0 <= spent_before < ceiling + ): + return [] + except (KeyError, TypeError, ValueError, OverflowError): + pass + return ["spent_before must be finite and below the stage ceiling"] + + +def allocation_problems(design: dict, stage: Stage) -> list[str]: + """Check the registered proportional cap split and three-times reserves.""" + problems = [] + for field in ("expected_costs", "caps", "reserves"): + values = design.get(field) + if not isinstance(values, dict) or set(values) != set(stage.models): + problems.append( + f"{field} must be an object with exactly the registered model keys" + ) + if problems: + return problems + expected = {} + for model in stage.models: + try: + raw_cost = design["expected_costs"][model] + cost = float(raw_cost) + if ( + type(raw_cost) not in (int, float) + or not math.isfinite(cost) + or cost <= 0 + ): + raise ValueError("expected cost is not finite and positive") + expected[model] = cost + except (KeyError, TypeError, ValueError, OverflowError): + problems.append(f"{model}: expected_costs must be finite and positive") + if problems: + return problems + try: + total = math.fsum(expected.values()) + except OverflowError: + total = math.inf + if not math.isfinite(total): + return ["the expected costs do not sum to a finite positive amount"] + stage_cap = WEB_CAP if stage.kind == "web" else CALCULATOR_CAP + for model, cost in expected.items(): + for field, want, tolerance, rule in ( + ("caps", stage_cap * cost / total, CAP_TOLERANCE, "split"), + ("reserves", 3 * cost / HOUSEHOLDS, RESERVE_TOLERANCE, "rule"), + ): + try: + raw_allocation = design[field][model] + got = float(raw_allocation) + if type(raw_allocation) not in (int, float) or not math.isfinite(got): + raise ValueError("allocation is not finite") + except (KeyError, TypeError, ValueError, OverflowError): + problems.append(f"{model}: {field} needs a finite allocation") + continue + if got < 0: + problems.append(f"{model}: {field} needs a nonnegative allocation") + continue + if abs(got - want) > tolerance: + name = "cap" if field == "caps" else "reserve" + problems.append(f"{model}: {name} ${got}, and the {rule} gives ${want}") + return problems + + +def registered_expected_costs(development_runs: Path, stage: Stage) -> dict[str, float]: + """Use exactly the report's development inventory and projection arithmetic. + + Calculator cheap models use all 320 finals in pe3/abl-* mctest folders; + frontier models use 10 pe/mctest finals. Web models use 100 finals per + arm across mctest and mafree, projecting the sum of the two arm means. + """ + from confirmatory_no_tools import read_regular, saved_children + + def development_paths(folder): + # Match the report's projection-only empty-inventory semantics. + try: + with os.scandir(folder) as entries: + return sorted( + Path(entry.path) + for entry in entries + if entry.name.endswith(".json") + ) + except (FileNotFoundError, NotADirectoryError): + return [] + + def costs(paths): + amounts = [] + for path in paths: + try: + record = json.loads(read_regular(path).decode("utf-8")) + amounts.append(attempt_cost(record)) + except ( + OSError, + ValueError, + TypeError, + AttributeError, + OverflowError, + ) as error: + raise ValueError(f"{path}: {error}") from error + return amounts + + expected = {} + for model in stage.models: + root = development_runs / model + if stage.kind == "web": + means = [] + for folder in ("web", "web-pe3"): + paths = [ + path + for board in ("mctest", "mafree") + for path in development_paths(root / folder / board / "households") + ] + if len(paths) != 100: + raise ValueError( + f"{model}/{folder}: {len(paths)} development finals, " + "expected 100" + ) + means.append(math.fsum(costs(paths)) / len(paths)) + expected[model] = math.fsum(means) * HOUSEHOLDS + else: + arms = ( + sorted( + p + for p in saved_children(root) + if p.name == "pe3" or p.name.startswith("abl-") + ) + if model in CHEAP + else [root / "pe"] + ) + paths = [ + path + for arm in arms + for path in development_paths(arm / "mctest" / "households") + ] + required = 320 if model in CHEAP else 10 + if len(paths) != required: + raise ValueError( + f"{model}: {len(paths)} development records for the cost " + f"projection, expected {required}" + ) + amounts = costs(paths) + expected[model] = ( + sum(amounts) + / len(amounts) + * HOUSEHOLDS + * (len(ARMS) if model in CHEAP else len(FRONTIER_ARMS)) + ) + return expected + + +def projection_problems(design: dict, stage: Stage) -> list[str]: + """Require declared expected costs to match the development projection.""" + try: + expected = registered_expected_costs(Path(design["development_runs"]), stage) + return [ + f"{model}: expected cost ${design['expected_costs'][model]}, " + f"and the development projection gives ${cost:.6f}" + for model, cost in expected.items() + if abs(float(design["expected_costs"][model]) - cost) > 1e-6 + ] + except ( + OSError, + KeyError, + ValueError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + ) as error: + return [f"the registered expected-cost projection: {error}"] + + +def anchor_clone_problems(command, stage: Stage) -> list[str]: + """Check only local anchor-clone metadata; never fetch or push.""" + arguments = [str(value) for value in command] if isinstance(command, list) else [] + repositories = [] + for index, argument in enumerate(arguments): + if argument == "--repo": + repositories.append( + arguments[index + 1] if index + 1 < len(arguments) else "" + ) + elif argument.startswith("--repo="): + repositories.append(argument.removeprefix("--repo=")) + if ( + len(repositories) != 1 + or not repositories[0] + or repositories[0].startswith("--") + ): + return ["the anchor command must name exactly one --repo clone"] + repo = Path(repositories[0]).expanduser() + + def git(*args): + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True + ) + + try: + tree = git("rev-parse", "--is-inside-work-tree") + head = git("rev-parse", "--verify", "HEAD^{commit}") + if tree.returncode or tree.stdout.strip() != "true" or head.returncode: + return [ + f"the anchor clone {repo}: needs a working tree " + "with a checked-out commit" + ] + problems = [] + branch = git("symbolic-ref", "--quiet", "HEAD") + expected = ANCHOR_REFS[stage.kind] + if branch.returncode or branch.stdout.strip() != expected: + problems.append( + f"the anchor clone {repo}: checked-out branch must be {expected}" + ) + origin = git("config", "--get", "remote.origin.url") + accepted = { + f"{prefix}{ANCHOR_REPOSITORY}{suffix}" + for prefix in ( + "https://github.com/", + "git@github.com:", + "ssh://git@github.com/", + ) + for suffix in ("", ".git") + } + if origin.returncode or origin.stdout.strip() not in accepted: + problems.append( + f"the anchor clone {repo}: origin must name {ANCHOR_REPOSITORY} " + "over SSH or HTTPS" + ) + return problems + except OSError as error: + return [f"the anchor clone {repo}: cannot read Git metadata: {error}"] + + +def restart_manifest_problems( + design: dict, stage: Stage, design_sha256: str | None +) -> list[str]: + """Bind every existing registered-model manifest to the input file's bytes.""" + from confirmatory_no_tools import read_regular, saved_children + + problems = [] + valid_hash = isinstance(design_sha256, str) and re.fullmatch( + r"[0-9a-f]{64}", design_sha256 + ) + for model in stage.models: + folder = Path(design["runs"]) / model + try: + paths = [ + path + for path in saved_children(folder) + if path.name.startswith("manifest-") and path.suffix == ".json" + ] + except ValueError as error: + problems.append(str(error)) + continue + for path in paths: + try: + manifest = json.loads(read_regular(path).decode("utf-8")) + except (OSError, ValueError, RecursionError) as error: + problems.append(f"{path}: cannot read restart manifest: {error}") + continue + if ( + not valid_hash + or not isinstance(manifest, dict) + or manifest.get("design_sha256") != design_sha256 + ): + problems.append( + f"{path}: design_sha256 is missing or differs from the design file" + ) + return problems + + +def preflight( + design: dict, models: tuple[str, ...], design_sha256: str | None = None +) -> list[str]: + """Return recorded input problems for every malformed design.""" + problems = [] + try: + if not isinstance(design, dict): + raise ValueError("the design must be an object") + return _preflight(design, models, design_sha256, problems) + except ( + OSError, + ValueError, + KeyError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + RuntimeError, + ) as error: + return [*problems, f"the design is malformed: {error}"] + + +def _preflight( + design: dict, + models: tuple[str, ...], + design_sha256: str | None = None, + problems: list[str] | None = None, +) -> list[str]: """What stops the run from starting.""" missing = [k for k in REQUIRED if k not in design] if missing: return [f"the design has no {', '.join(missing)}"] - problems = [] + try: + stage = stage_for(design) + except ValueError as error: + return [str(error)] + if stage.kind == "calculator" and "frontier_arms" not in design: + return ["the design has no frontier_arms"] + if problems is None: + problems = [] + problems.extend(restart_manifest_problems(design, stage, design_sha256)) + if ( + stage.kind == "calculator" + and design.get("overall_cap") != CALCULATOR_OVERALL_CAP + ): + problems.append("the calculator design's overall_cap must be 150") + problems.extend(stage_start_problems(design)) + allocation = allocation_problems(design, stage) + problems.extend(allocation) + if not allocation: + problems.extend(projection_problems(design, stage)) from confirmatory_no_tools import ( BOARD_INPUT_ERRORS, _board_input, @@ -2180,13 +3089,25 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: ) problems.extend(stage_problems(design)) + if stage.kind == "web": + problems.extend(web_preflight(design)) + if any(pilot.model_profile(model).provider == "gemini" for model in models): + try: + version = importlib.metadata.version("litellm") + except importlib.metadata.PackageNotFoundError: + version = "not installed" + if version != pilot.GEMINI_SEARCH_PATCH_LITELLM: + problems.append( + f"litellm {version} is installed; the Gemini search patch was " + f"checked against {pilot.GEMINI_SEARCH_PATCH_LITELLM} only" + ) try: no_tools_root = Path(design["no_tools"]["runs"]).resolve() calculator_root = Path(design["runs"]).resolve() if no_tools_root.is_relative_to( calculator_root ) or calculator_root.is_relative_to(no_tools_root): - problems.append("the no-tools and calculator runs roots overlap") + problems.append(f"the no-tools and {stage.kind} runs roots overlap") except (KeyError, TypeError, OSError, ValueError, RuntimeError) as error: problems.append(f"the no-tools runs root is invalid: {error}") command = design["anchor_command"] @@ -2196,8 +3117,8 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: or shutil.which(str(command[0])) is None ): problems.append(f"the anchor command {command!r} is not a command here") - if design.get("stated_engine_overrides"): - problems.append("the confirmatory run states the board's engine: no overrides") + problems.extend(anchor_clone_problems(command, stage)) + problems.extend(rehearsal_problems(design)) preregistration = Path(design["preregistration"]) try: if ( @@ -2277,11 +3198,15 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: installed = pilot._versions()["policyengine-us"] if installed != engine: problems.append(f"policyengine-us {installed} is installed, not {engine}") - if set(design["arms"]) != set(ARMS) or set(design["frontier_arms"]) != set( - FRONTIER_ARMS + mappings = { + stage.folder_keys[model]: stage.model_arms[model] for model in stage.models + } + if any(set(design[key]) != set(labels) for key, labels in mappings.items()) or ( + stage.kind == "web" + and any(key in design for key in ("frontier_arms", "frontier_models")) ): problems.append("the design's arms are not the registered ones") - for name in ("arms", "frontier_arms"): + for name in mappings: if len(set(design[name].values())) != len(design[name]): problems.append(f"two {name} share a folder") @@ -2291,9 +3216,7 @@ def household_paths(stage): return saved_json_paths(stage / "households") for model in models: - if model not in design["caps"] or model not in design["reserves"]: - problems.append(f"{model}: no cap or reserve") - for label in ARMS if model in CHEAP else FRONTIER_ARMS: + for label in stage.arms_for(model): try: arm_profile(model, label) except (ValueError, SystemExit) as error: @@ -2345,10 +3268,33 @@ def main(argv=None) -> int: resume.add_argument("model") resume.add_argument("--note", default="the account was fixed") args = parser.parse_args(argv) - design = json.loads(args.design.read_text()) + refusal_prefix = ( + "No resume requested" if args.command == "resume" else "The run cannot start" + ) + try: + design_bytes = args.design.read_bytes() + design = json.loads(design_bytes.decode("utf-8")) + design_sha256 = sha256_bytes(design_bytes) + if not isinstance(design, dict): + raise ValueError("the design must be an object") + stage = stage_for(design) + except (OSError, ValueError, TypeError, AttributeError, RecursionError) as error: + print(f"{refusal_prefix}: {error}") + return 1 if args.command == "resume": - folder = Path(design["runs"]) / args.model - write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) + try: + if args.model not in stage.models: + raise ValueError( + f"model {args.model!r} is not in the {stage.kind} stage" + ) + runs = design.get("runs") + if not isinstance(runs, str) or not runs.strip(): + raise ValueError("the design must name a usable runs folder") + folder = Path(runs) / args.model + write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) + except (OSError, ValueError, TypeError) as error: + print(f"No resume requested: {error}") + return 1 print(f"{args.model}: resume requested; the runner resumes within a second") return 0 if args.workers is not None and args.workers != design.get("workers"): @@ -2357,8 +3303,11 @@ def main(argv=None) -> int: f"runs {design.get('workers')} engine workers per model." ) return 1 - models = CHEAP + FRONTIER - problems = preflight(design, models) + models = stage.models + problems = preflight(design, models, design_sha256) + if problems: + print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) + return 1 from confirmatory_no_tools import BOARD_INPUT_ERRORS board = Path(design["board"]) @@ -2372,7 +3321,7 @@ def main(argv=None) -> int: if problems: print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) return 1 - plan = Run(design) + plan = Run(design, design_sha256=design_sha256) if args.dry_run: for model, lanes in plan.lanes.items(): sizes = [len(lanes.households(lane)) for lane in (1, 2)] diff --git a/tests/fake_anchor.py b/tests/fake_anchor.py index 73b5c32f..5ee668ca 100644 --- a/tests/fake_anchor.py +++ b/tests/fake_anchor.py @@ -1,13 +1,17 @@ """The confirmatory runner's anchor command in its tests: a copy of the run's anchors.jsonl to FOLDER/anchored.jsonl. -Usage: python fake_anchor.py FOLDER ANCHORS +Usage: python fake_anchor.py [--repo CLONE] FOLDER ANCHORS """ +import argparse import shutil -import sys from pathlib import Path -folder, anchors = Path(sys.argv[1]), Path(sys.argv[2]) -shutil.copyfile(anchors, folder / "anchored.jsonl") +parser = argparse.ArgumentParser() +parser.add_argument("--repo", type=Path) +parser.add_argument("folder", type=Path) +parser.add_argument("anchors", type=Path) +args = parser.parse_args() +shutil.copyfile(args.anchors, args.folder / "anchored.jsonl") print("anchored") diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index c7361703..d764ad9c 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -81,19 +81,23 @@ def test_engine_metadata_refuses_full_calculator_start( @pytest.mark.parametrize("fault", ["malformed", "unreadable"]) -def test_main_names_engine_reread_failure_and_preserves_all_problems( +def test_main_names_an_engine_reread_failure_after_a_clean_preflight( world, tmp_path, monkeypatch, capsys, fault ): w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) - # Isolate main's final engine reread, which must retain preflight's problems. - monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + # Isolate main's final engine reread. Main reads the board only after a + # clean preflight, as test_confirmatory_run.py requires in + # test_command_line_prints_preflight_refusals_before_reading_the_board, + # so preflight here reports nothing. + monkeypatch.setattr(cr, "preflight", lambda *_: []) design_path = tmp_path / "design.json" design_path.write_text(json.dumps(design)) assert cr.main(["run", str(design_path)]) == 1 output = capsys.readouterr().out - assert "The run cannot start" in output - assert "earlier preflight problem" in output - assert str(metadata) in output + assert "The run cannot start:\n" in output, output + assert f"- {metadata}: the board's engine check failed: " in output, ( + "main's final engine reread names the board's metadata and its failure" + ) assert not w.provider.count @@ -126,17 +130,39 @@ def test_preflight_refuses_an_installed_engine_other_than_the_boards( assert not w.provider.count -def test_main_rechecks_the_installed_engine_and_preserves_all_problems( +def test_main_rechecks_the_installed_engine_after_a_clean_preflight( world, tmp_path, monkeypatch, capsys ): w, design = installed_elsewhere(world, tmp_path, monkeypatch) - # Isolate main's final engine check, which must retain preflight's problems. - monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + # Isolate main's final engine check, which runs only after a clean preflight. + monkeypatch.setattr(cr, "preflight", lambda *_: []) design_path = tmp_path / "design.json" design_path.write_text(json.dumps(design)) assert cr.main(["run", str(design_path)]) == 1 output = capsys.readouterr().out - assert "The run cannot start" in output - assert "earlier preflight problem" in output - assert "the board's engine check failed: policyengine-us 2.41.0" in output + assert "The run cannot start:\n" in output, output + assert "the board's engine check failed: policyengine-us 2.41.0" in output, ( + "main's final engine check refuses an installed engine other than the board's" + ) + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable", "installed-elsewhere"]) +def test_main_prints_every_preflight_problem_and_starts_nothing( + world, tmp_path, monkeypatch, capsys, fault +): + if fault == "installed-elsewhere": + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + else: + w, design, _ = engine_fault(world, tmp_path, monkeypatch, fault) + problems = ["earlier preflight problem", "a second preflight problem"] + monkeypatch.setattr(cr, "preflight", lambda *_: list(problems)) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start:\n" in output, output + assert all(f"- {problem}\n" in output for problem in problems), ( + "main prints every preflight problem" + ) assert not w.provider.count diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index e85e7562..df9994f9 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -425,20 +425,22 @@ def test_preflight_refuses_a_design_without_spent_before(tmp_path, monkeypatch): @pytest.mark.parametrize( - "declared, refused", - [("1", False), ("0.1", True), (True, False), (False, True)], + "declared", + ["1", "0.1", True, False], ids=["text-enough", "text-short", "true", "false"], ) -def test_preflight_counts_numeric_text_and_booleans_as_numbers( - tmp_path, monkeypatch, declared, refused +def test_preflight_refuses_numeric_text_and_booleans_as_spend( + tmp_path, monkeypatch, declared ): - """As the report does: float() decides what a spent_before is worth.""" + """A declaration must be a nonboolean JSON number before conversion.""" world, design = pool_design(tmp_path, monkeypatch) saved_spend(world, design) + assert nt.stage_problems(design) == [] design["spent_before"] = declared - problems = cr.preflight(design, (cr.CHEAP[0],)) - assert any(p.startswith("no-tools spend") for p in problems) is refused, problems - assert not any("finite, nonnegative" in p for p in problems), problems + problems = nt.stage_problems(design) + assert len(problems) == 1 and "finite, nonnegative" in problems[0], ( + "saved no-tools spend refuses nonnumeric or boolean declarations" + ) @pytest.mark.parametrize( diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 81861839..23443fef 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -46,10 +46,10 @@ TZ = "America/New_York" PREREG = "A frozen pre-registration for a test.\n" ENGINE = "2.60.4" -# The harness times requests with the raw wall clock, which steps back by -# about 0.1 ms on this host, even within a thread; the report allows a second, as -# here, which also covers a small correction of the clock. -CLOCK_SLACK = 1.0 +# The harness times requests with the raw wall clock. As in the report, allow +# 0.1 s plus the largest backward step measured from the host's sampled wall +# and monotonic clocks. +CLOCK_SLACK = 0.1 # The anchor command in a test: a copy of the anchors to the test's folder. ANCHOR_SCRIPT = Path(__file__).resolve().parent / "fake_anchor.py" # How far apart anchors and their triggers' stops may be in a test, on a @@ -58,6 +58,21 @@ TRIGGER_BOUND = 5.0 +@pytest.fixture(autouse=True) +def keep_litellm_gemini_config(monkeypatch): + """Restore the class descriptors after every test, including fake web runs.""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + for name in ( + "_resolve_search_tool_conflict", + "_drop_search_tools_mixed_with_functions", + ): + monkeypatch.setattr(VertexGeminiConfig, name, VertexGeminiConfig.__dict__[name]) + return VertexGeminiConfig + + # ---------------------------------------------------------------- streams @@ -128,6 +143,183 @@ def test_an_arm_whose_features_are_not_registered_is_refused(monkeypatch): cr.arm_profile("gpt-6-luna", "loo-flat") +def test_stage_table_registers_the_calculator_and_web_models_arms_and_folders(): + calculator, web = cr.STAGES["calculator"], cr.STAGES["web"] + assert calculator.models == cr.CHEAP + cr.FRONTIER + assert calculator.model_arms == { + m: cr.ARMS if m in cr.CHEAP else cr.FRONTIER_ARMS for m in calculator.models + } + assert calculator.folder_keys == { + m: "arms" if m in cr.CHEAP else "frontier_arms" for m in calculator.models + } + assert calculator.order_stream == "order/{model}" + assert calculator.profile_rule == "calculator" + assert calculator.arms_for("gemini-3.1-pro-preview") == cr.FRONTIER_ARMS + assert calculator.folder_key_for("gemini-3.1-pro-preview") == "frontier_arms" + assert web.models == ("gpt-6-luna", "gemini-3.5-flash-lite", "claude-sonnet-5.5") + assert web.model_arms == {m: ("web", "web+pe3") for m in web.models} + assert web.folder_keys == {m: "arms" for m in web.models} + assert web.order_stream == "order/web/{model}" + assert web.profile_rule == "web" + with pytest.raises(KeyError, match="no model in the web stage"): + web.arms_for("gemini-3.1-pro-preview") + with pytest.raises(KeyError, match="no model in the web stage"): + web.folder_key_for("gemini-3.1-pro-preview") + + +@pytest.mark.parametrize("kind", [None, "", "unknown", [], 7]) +def test_design_kind_is_required_and_registered_before_any_stage_starts(kind): + design = {} if kind is None else {"kind": kind} + with pytest.raises(ValueError, match="kind must be calculator or web"): + cr.stage_for(design) + with pytest.raises(ValueError, match="kind must be calculator or web"): + cr.Run(design, design_sha256="0" * 64) + + +@pytest.mark.parametrize("label", cr.WEB_ARMS) +@pytest.mark.parametrize("model", cr.WEB_MODELS) +def test_web_arm_profile_keeps_search_the_benchmark_rule_and_its_calculator( + label, model +): + profile = cr.arm_profile(model, label) + assert profile.arm == label and profile.uses_web + assert profile.benchmark_search_rule is True + assert profile.prompt_version == cr.PROMPT_VERSION + assert profile.uses_calculator == (label == "web+pe3") + assert set(profile.tool_features) == ( + set(pilot.TOOL_FEATURES) if profile.uses_calculator else set() + ) + + +def first_gemini_http_body(monkeypatch, label): + """Run the harness's initial request through the real LiteLLM HTTP path.""" + import httpx + + class BodyCaptured(BaseException): + pass + + captured = [] + + def capture(self, request, **kwargs): + assert request.method == "POST" + assert request.url.host == "generativelanguage.googleapis.com" + captured.append(json.loads(request.content)) + raise BodyCaptured + + monkeypatch.setenv("GEMINI_API_KEY", "synthetic-no-network") + monkeypatch.setattr(httpx.Client, "send", capture) + profile = cr.arm_profile("gemini-3.5-flash-lite", label) + # run_household calls build_initial_request and assembles the actual first + # turn, including submit_outputs and the arm's calculator/search tools. + with pytest.raises(BodyCaptured): + pilot.run_household( + scenario("h00"), + VARIABLES, + lambda arguments: pytest.fail("the captured request cannot call a tool"), + cr.default_completion(profile), + pilot.Spend(1.0), + profile=profile, + ) + assert len(captured) == 1 + return captured[0] + + +@pytest.mark.parametrize("label", cr.WEB_ARMS) +def test_gemini_web_http_body_keeps_search_beside_function_declarations( + monkeypatch, label, world +): + world(kind="web", models=("gemini-3.5-flash-lite",)).execute() + body = first_gemini_http_body(monkeypatch, label) + assert any("googleSearch" in tool for tool in body["tools"]), ( + "the patched Gemini HTTP body must offer googleSearch" + ) + functions = { + function["name"] + for tool in body["tools"] + for function in tool.get("function_declarations", []) + } + assert functions == ( + {"submit_outputs", "calculate_household"} + if label == "web+pe3" + else {"submit_outputs"} + ), "the Gemini HTTP body must retain every function declaration" + + +@pytest.mark.parametrize("label", cr.WEB_ARMS) +def test_unpatched_gemini_web_http_body_drops_search_beside_functions( + monkeypatch, label, keep_litellm_gemini_config +): + config = keep_litellm_gemini_config + original = { + name: config.__dict__[name] + for name in ( + "_resolve_search_tool_conflict", + "_drop_search_tools_mixed_with_functions", + ) + } + pilot.allow_gemini_search_with_functions() + for name, descriptor in original.items(): + monkeypatch.setattr(config, name, descriptor) + body = first_gemini_http_body(monkeypatch, label) + assert not any("googleSearch" in tool for tool in body["tools"]), ( + "the unpatched control must expose LiteLLM's removal of googleSearch" + ) + assert any("function_declarations" in tool for tool in body["tools"]), ( + "the unpatched control must still reach Gemini's function declarations" + ) + + +@pytest.mark.parametrize( + "label,changed,message", + [ + ("web", {"uses_web": False}, "no web search"), + ("web+pe3", {"uses_web": False}, "no web search"), + ("web", {"uses_calculator": True}, "gives a calculator"), + ("web+pe3", {"uses_calculator": False}, "full calculator features"), + ("web", {"benchmark_search_rule": False}, "benchmark-search rule"), + ] + + [ + ( + "web+pe3", + {"tool_features": set(pilot.TOOL_FEATURES) - {feature}}, + "full calculator features", + ) + for feature in pilot.TOOL_FEATURES + ], +) +def test_web_arm_profile_refuses_each_wrong_profile_alone( + monkeypatch, label, changed, message +): + from types import SimpleNamespace + + original = cr.arm_profile("gpt-6-luna", label) + values = { + name: getattr(original, name) + for name in ( + "uses_web", + "uses_calculator", + "tool_features", + "benchmark_search_rule", + ) + } + monkeypatch.setattr( + pilot, "model_profile", lambda *a, **k: SimpleNamespace(**{**values, **changed}) + ) + with pytest.raises(ValueError, match=message): + cr.arm_profile("gpt-6-luna", label) + + +def test_web_orders_use_the_web_stream_for_every_drawn_household(): + got = cr.orders(9, cr.WEB_MODELS[0], POOL, cr.WEB_ARMS, kind="web") + generator = cr.stream(9, "order/web/gpt-6-luna") + assert got == { + sid: [str(a) for a in generator.permutation(list(cr.WEB_ARMS))] for sid in POOL + } + assert cr.orders(9, cr.WEB_MODELS[0], POOL[:3], cr.WEB_ARMS, kind="web") == { + sid: got[sid] for sid in POOL[:3] + } + + def litellm_names(groups) -> set[str]: """The names of litellm's exception classes under any of `groups`.""" import inspect @@ -553,13 +745,38 @@ def test_the_clock_never_goes_back(monkeypatch): monkeypatch.setattr(time, "time", lambda: next(readings)) assert [cr.clock() for _ in range(5)] == [ base, - base, - base + 2e-4, + math.nextafter(base, math.inf), base + 2e-4, + math.nextafter(base + 2e-4, math.inf), base + 3e-4, ] +def test_a_frozen_wall_clock_is_strictly_increasing_across_threads(monkeypatch): + base = time.time() + 1000.0 + monkeypatch.setattr(cr, "_clock_last", 0.0) + monkeypatch.setattr(time, "time", lambda: base) + readings = [[] for _ in range(8)] + ready = threading.Barrier(len(readings)) + + def read(values): + ready.wait() + values.extend(cr.clock() for _ in range(100)) + + threads = [threading.Thread(target=read, args=(values,)) for values in readings] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + values = [value for mine in readings for value in mine] + assert len(values) == 800 and len(set(values)) == len(values), ( + "runner clock values are strictly increasing across threads rule" + ) + assert all(all(a < b for a, b in zip(mine, mine[1:])) for mine in readings), ( + "runner clock values are strictly increasing across threads rule" + ) + + def test_a_backward_wall_clock_does_not_delay_a_due_anchor(monkeypatch): """Once the ordered clock reaches the due time, a wall-clock step cannot make the anchor not due again.""" @@ -595,6 +812,88 @@ def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch assert len(w.attempts(model)) == N_DRAW * len(labels_of(model)) +@pytest.mark.parametrize("after", [0.1, 0.25]) +def test_a_two_second_wall_clock_step_keeps_the_run_procedure( + world, monkeypatch, after +): + monkeypatch.setattr(cr, "_clock_last", 0.0) + model = "gpt-6-sol" + w = world(models=(model,)) + drawn = cr.orders(w.run.master, model, w.draw_order, labels_of(model)) + initial = {sid: drawn[sid][0] for sid in w.draw_order[:2]} + admitted = threading.Barrier(2) + for sid, arm in initial.items(): + # Both first households must be under way before either can finish. + w.provider.specs[(model, arm, sid, 1)] = { + "check": lambda _: admitted.wait(timeout=120) + } + real_time, make_host, make_pool = time.time, w.run.host_factory, w.run.pool_factory + real_end = cr.Lanes.end_attempt + started = [] + readings_after = [] + state_lock = threading.Lock() + state = {"ticks": 0, "ended": 0, "recovered": False} + + def start_host(folder): + host = make_host(folder) + # Time the step from a sampled run, so a slow startup cannot hide it. + # This unused pre-step reading also fixes the clock's high-water mark + # when host construction or scheduling took longer than two seconds. + before = cr.clock() + started.append((time.monotonic(), before)) + return host + + def stepped_time(): + now = real_time() + with state_lock: + if not started: + return now + elapsed = time.monotonic() - started[0][0] + if elapsed < after: + return started[0][1] + elapsed + readings_after.append(now) + if state["recovered"]: + return state["recovery_wall"] + now - state["recovery_real"] + # Advance synthetic time while keeping the first overlap inside + # the two-second correction, even if this host pauses for a minute. + state["ticks"] += 1 + return started[0][1] + after - 2.0 + state["ticks"] * 1e-6 + + def start_pool(lanes): + remaining = after - (time.monotonic() - started[0][0]) + if remaining > 0: + time.sleep(remaining) + stepped_time() + w.run.host.sample() # retain both sides of the correction in real evidence + return make_pool(lanes) + + def end_attempt(lanes, sid, arm, n, *args, **kwargs): + result = real_end(lanes, sid, arm, n, *args, **kwargs) + if n == 1 and sid in initial: + with state_lock: + state["ended"] += 1 + if state["ended"] == len(initial) * len(labels_of(model)): + state["recovery_real"] = real_time() + state["recovery_wall"] = started[0][1] + after + 0.1 + state["recovered"] = True + return result + + w.run.host_factory = start_host + w.run.pool_factory = start_pool + monkeypatch.setattr(cr.Lanes, "end_attempt", end_attempt) + monkeypatch.setattr(time, "time", stepped_time) + w.execute() + assert readings_after, "the two-second wall-clock step must occur during the run" + assert host_from_files(w.runs() / "host").back > 1.9 + problems = run_problems(w) + assert problems == [], ( + f"runner procedure survives a two-second wall-clock step rule: {problems}" + ) + assert len(w.attempts(model)) == N_DRAW * len(labels_of(model)), ( + "runner procedure survives a two-second wall-clock step rule" + ) + + def test_the_ceiling_crosses_at_its_amount_and_stays_crossed(): ceiling = cr.Ceiling(1.0, 0.25) ceiling.add(0.5) @@ -766,6 +1065,7 @@ def __init__( no_output=(), models=None, pool=FakePool, + kind="calculator", ): self.tmp = tmp prereg = tmp / "prereg.md" @@ -795,12 +1095,15 @@ def __init__( monkeypatch.setitem( pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] ) - self.models = tuple(models or cr.CHEAP + cr.FRONTIER) + self.kind = kind + self.models = tuple(models or cr.STAGES[kind].models) budget_models = tuple(dict.fromkeys(cr.CHEAP + cr.FRONTIER + self.models)) self.design = { + "kind": kind, "preregistration": str(prereg), "frozen_sha256": frozen, "runs": str(tmp / "runs"), + "development_runs": str(tmp / "development-runs"), "stage": "confirm", "board": str(board_path), "engine": f"policyengine-us {ENGINE}", @@ -814,10 +1117,14 @@ def __init__( "workers": 2, "anchor_command": [sys.executable, str(ANCHOR_SCRIPT), str(tmp)], } + if kind == "web": + self.design["arms"] = {a: a.replace("+", "-") for a in cr.WEB_ARMS} + del self.design["frontier_arms"] self.readers = Readers() self.provider = FakeProvider(self, specs) self.run = cr.Run( self.design, + design_sha256=cr.sha256_bytes(json.dumps(self.design).encode("utf-8")), models=self.models, converse=self.provider, pool_factory=pool, @@ -827,6 +1134,10 @@ def __init__( ) def execute(self): + self.design_path = self.tmp / "design.json" + data = json.dumps(self.design).encode("utf-8") + self.design_path.write_bytes(data) + self.run.design_sha256 = cr.sha256_bytes(data) self.run.execute() return self @@ -852,9 +1163,7 @@ def events(self, model, kind=None) -> list[dict]: ] def folder(self, model, label) -> Path: - arms = ( - self.design["arms"] if model in cr.CHEAP else self.design["frontier_arms"] - ) + arms = self.design[cr.STAGES[self.kind].folder_key_for(model)] return self.runs() / model / arms[label] / "confirm" / "households" @@ -869,6 +1178,11 @@ def host_from_files(folder: Path): samples = [ json.loads(x) for x in (folder / "host-log.jsonl").read_text().splitlines() if x ] + back = 0.0 + for first, second in zip(samples, samples[1:]): + elapsed = second["monotonic"] - first["monotonic"] + if elapsed >= 0: + back = max(back, elapsed - (second["time"] - first["time"])) spans = cr.guard_spans( (folder / "clamshell-guard.log").read_text(), meta["timezone"], end ) @@ -882,6 +1196,9 @@ def host_from_files(folder: Path): pressure = [float(s["time"]) for s in good if int(s["pressure"]) > 1] class Host: + def __init__(self): + self.back = back + def trouble(self, s, e): for a, b, what in spans: if a <= e and b >= s: @@ -962,6 +1279,7 @@ def run_problems(world: World) -> list[str]: """Every rule of the run procedure, recomputed from the runner's files.""" problems: list[str] = [] host = host_from_files(world.runs() / "host") + clock_slack = CLOCK_SLACK + host.back frozen = world.design["frozen_sha256"] problems += chain_problems(world.runs() / "host" / "host-log.jsonl", cr.HOST_SEED) problems += anchor_problems(world) @@ -971,8 +1289,8 @@ def run_problems(world: World) -> list[str]: all_attempts = [] ceiling_events = {} for model in world.models: - labels = labels_of(model) - drawn = cr.orders(master, model, world.draw_order, labels) + labels = cr.STAGES[world.kind].arms_for(model) + drawn = cr.orders(master, model, world.draw_order, labels, kind=world.kind) lines = world.attempts(model) events = world.events(model) for name in ("run-log.jsonl", "admission-log.jsonl"): @@ -1058,8 +1376,8 @@ def run_problems(world: World) -> list[str]: for key in ("request_started_at", "request_completed_at") ] if times and ( - min(times) < x["start"] - CLOCK_SLACK - or max(times) > x["end"] + CLOCK_SLACK + min(times) < x["start"] - clock_slack + or max(times) > x["end"] + clock_slack ): problems.append( f"{at}#{k}: requests {min(times)!r}-{max(times)!r} outside " @@ -1265,6 +1583,9 @@ def test_an_untouched_run_runs_every_household_and_arm(world): manifests = list((w.runs() / model).glob("manifest-*.json")) assert len(manifests) == 1 m = json.loads(manifests[0].read_text()) + assert m["design_sha256"] == cr.sha256_file(w.design_path), ( + "every manifest binds the exact design file bytes rule" + ) assert m["arms"] == { label: { "arm": cr.arm_string(label), @@ -2071,7 +2392,9 @@ def before_rerun(lanes): def test_the_worker_count_is_the_design_s(tmp_path, capsys): design = tmp_path / "design.json" - design.write_text(json.dumps({"runs": str(tmp_path / "runs"), "workers": 2})) + design.write_text( + json.dumps({"kind": "calculator", "runs": str(tmp_path / "runs"), "workers": 2}) + ) assert cr.main(["run", str(design), "--workers", "3"]) == 1 assert "the design runs 2 engine workers" in capsys.readouterr().out @@ -2203,6 +2526,154 @@ def test_the_preflight_names_a_missing_field(): assert cr.preflight({}, cr.CHEAP) == [f"the design has no {', '.join(cr.REQUIRED)}"] +INVALID_SAVED_COSTS = [ + "missing-totals", + "missing-cost", + None, + False, + True, + "0", + math.nan, + -0.01, +] + + +def invalidate_saved_cost(record, fault): + if fault == "missing-totals": + record.pop("totals") + elif fault == "missing-cost": + record["totals"].pop("cost_usd") + else: + record["totals"]["cost_usd"] = fault + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("record_kind", ["final", "replaced", "stub"]) +@pytest.mark.parametrize("fault", INVALID_SAVED_COSTS) +def test_saved_attempt_cost_requires_an_explicit_valid_total(kind, record_kind, fault): + record = { + "scenario_id": "last-household", + "kind": kind, + "status": "harness_error" if record_kind == "stub" else "complete", + "attempts": [], + "totals": {"cost_usd": 0.0}, + } + assert cr.attempt_cost(record) == 0.0 + invalidate_saved_cost(record, fault) + refused = False + try: + cr.attempt_cost(record) + except ValueError: + refused = True + assert refused, ( + "saved attempt costs require a present nonboolean finite nonnegative number" + ) + + +def test_live_attempt_cost_keeps_recording_a_missing_total(): + assert cr.attempt_cost({}, saved=False) == 0.0, ( + "live accounting retains records before saved-evidence validation" + ) + + +def synthetic_development_tree(root, kind, expected): + """Only invented costs; retain the registered development folder counts.""" + stage = cr.STAGES[kind] + for model in stage.models: + if kind == "web": + unit = expected[model] / (2 * cr.HOUSEHOLDS) + groups = [ + (root / model / arm / board / "households", 50) + for arm in ("web", "web-pe3") + for board in ("mctest", "mafree") + ] + else: + unit = expected[model] / (cr.HOUSEHOLDS * len(stage.arms_for(model))) + arms = ( + ["pe3", *(f"abl-{n:02}" for n in range(15))] + if model in cr.CHEAP + else ["pe"] + ) + groups = [ + ( + root / model / arm / "mctest/households", + 20 if model in cr.CHEAP else 10, + ) + for arm in arms + ] + for folder, count in groups: + folder.mkdir(parents=True, exist_ok=True) + for n in range(count): + (folder / f"synthetic-{n:03}.json").write_text( + json.dumps({"totals": {"cost_usd": unit}}) + ) + + +def synthetic_anchor_clone(tmp_path, kind): + """A local clone with synthetic history and registered remote metadata.""" + branches = { + "calculator": "anchors/v2-confirmatory", + "web": "anchors/v2-confirmatory-web", + } + source = tmp_path.parent / "anchor-fixture-source" + if not source.exists(): + subprocess.run( + [ + "git", + "init", + "--quiet", + "--template=", + "--initial-branch=fixture", + str(source), + ], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(source), + "-c", + "user.name=Anchor fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "core.hooksPath=/dev/null", + "-c", + "commit.gpgsign=false", + "commit", + "--quiet", + "--allow-empty", + "-m", + "Create synthetic anchor history.", + ], + check=True, + ) + repo = tmp_path / f"anchor-clone-{kind}" + if not repo.exists(): + subprocess.run( + ["git", "clone", "--quiet", "--shared", str(source), str(repo)], + check=True, + ) + subprocess.run( + ["git", "-C", str(repo), "checkout", "--quiet", "-b", branches[kind]], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(repo), + "remote", + "set-url", + "origin", + "https://github.com/PolicyEngine/policybench-v2-anchors.git", + ], + check=True, + ) + return repo + + def complete_preflight_design(w, tmp_path): """Each calculator preflight starts after standing no-tools finals that pass the no-tools script's treatment checks: built through the harness from the @@ -2236,6 +2707,10 @@ def complete_preflight_design(w, tmp_path): baseline.write_text('{"exclusions": []}\n') design = dict( w.design, + overall_cap=150.0, + expected_costs={m: 1.0 for m in cr.STAGES["calculator"].models}, + caps={m: 145.0 / 6 for m in cr.STAGES["calculator"].models}, + reserves={m: 3.0 / cr.HOUSEHOLDS for m in cr.STAGES["calculator"].models}, eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, baseline_exclusions={"path": str(baseline), "sha256": cr.sha256_file(baseline)}, treatment_harness=str(cr.ROOT), @@ -2245,6 +2720,17 @@ def complete_preflight_design(w, tmp_path): "folder": "none", "stage": "confirm", }, + development_runs=str(tmp_path / "development-runs"), + anchor_command=[ + sys.executable, + str(ANCHOR_SCRIPT), + "--repo", + str(synthetic_anchor_clone(tmp_path, "calculator")), + str(tmp_path), + ], + ) + synthetic_development_tree( + Path(design["development_runs"]), "calculator", design["expected_costs"] ) for model in cr.CHEAP: for sid in design["draw_order"]: @@ -2261,6 +2747,28 @@ def complete_preflight_design(w, tmp_path): return design +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("fault", INVALID_SAVED_COSTS) +def test_preflight_refuses_invalid_saved_no_tools_cost_on_the_last_household( + world, tmp_path, monkeypatch, kind, fault +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + path = ( + Path(design["no_tools"]["runs"]) + / cr.CHEAP[-1] + / "none/confirm/households" + / f"{w.draw_order[-1]}.json" + ) + record = json.loads(path.read_text()) + invalidate_saved_cost(record, fault) + path.write_text(json.dumps(record)) + problems = cr.preflight(design, models) + assert problems and any( + "cost" in problem or "totals" in problem for problem in problems + ), "preflight refuses invalid saved costs even on a model's final household" + + def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( world, tmp_path, monkeypatch ): @@ -2372,150 +2880,1780 @@ def test_preflight_refuses_overlapping_no_tools_and_calculator_roots( def test_resume_writes_the_operator_s_note(tmp_path): design = tmp_path / "design.json" - design.write_text(json.dumps({"runs": str(tmp_path / "runs")})) + design.write_text( + json.dumps({"kind": "calculator", "runs": str(tmp_path / "runs")}) + ) assert cr.main(["resume", str(design), "gpt-6-luna", "--note", "key rotated"]) == 0 note = json.loads((tmp_path / "runs" / "gpt-6-luna" / "RESUME.json").read_text()) assert note["note"] == "key rotated" -# ---------------------------------------------------------------- the harness - - -def chat_response(calls): +def complete_web_design(world, tmp_path, monkeypatch, *, archives=False, validate=True): + """A web preflight with a genuinely ended synthetic calculator stage.""" from types import SimpleNamespace - message = SimpleNamespace( - role="assistant", content=None, tool_calls=calls, function_call=None + w = world( + overall_cap=150.0, + caps={m: 145.0 / 6 for m in cr.STAGES["calculator"].models}, + reserve=3.0 / cr.HOUSEHOLDS, ) - usage = SimpleNamespace( - prompt_tokens=1000, - completion_tokens=200, - total_tokens=1200, - reasoning_tokens=0, - prompt_tokens_details=SimpleNamespace(cached_tokens=0), - completion_tokens_details=None, + if archives: + model, sid = cr.CHEAP[0], w.draw_order[0] + w.provider.specs = { + (model, "pe", sid, n): { + "status": "provider_error", + "error": "RateLimitError: synthetic", + } + for n in (1, 2) + } + w.design.update(complete_preflight_design(w, tmp_path)) + w.execute() + calculator = tmp_path / "calculator-design.json" + calculator.write_text(json.dumps(w.design)) + design = copy.deepcopy(w.design) + design.update( + kind="web", + runs=str(tmp_path / "web-runs"), + arms={a: a.replace("+", "-") for a in cr.WEB_ARMS}, + calculator={"path": str(calculator), "sha256": cr.sha256_file(calculator)}, + expected_costs=dict(zip(cr.WEB_MODELS, (3.0, 5.0, 12.0))), + caps=dict(zip(cr.WEB_MODELS, (18.0, 30.0, 72.0))), + reserves={ + m: 3.0 * cost / cr.HOUSEHOLDS + for m, cost in zip(cr.WEB_MODELS, (3.0, 5.0, 12.0)) + }, + overall_cap=270.0, + spent_before=math.fsum( + [w.design["spent_before"], *cr.calculator_evidence(w.design)] + ), + anchor_command=[ + sys.executable, + str(ANCHOR_SCRIPT), + "--repo", + str(synthetic_anchor_clone(tmp_path, "web")), + str(tmp_path), + ], ) - return SimpleNamespace( - choices=[SimpleNamespace(message=message, finish_reason="tool_calls")], - usage=usage, - model="claude-haiku-5-5", + del design["frontier_arms"] + synthetic_development_tree( + Path(design["development_runs"]), "web", design["expected_costs"] + ) + monkeypatch.setattr( + pilot, "_git", lambda *args: "0" * 40 if args[0] == "rev-parse" else "" ) + real = subprocess.run + def git_pins(args, *a, **kw): + if list(args) == ["git", "-C", str(cr.ROOT), "rev-parse", "HEAD"]: + return SimpleNamespace(returncode=0, stdout=cr.TREATMENT_COMMIT + "\n") + return real(args, *a, **kw) -def chat_call(name, arguments, call_id): - from types import SimpleNamespace + monkeypatch.setattr(subprocess, "run", git_pins) + import policybench - return SimpleNamespace( - id=call_id, - type="function", - function=SimpleNamespace(name=name, arguments=json.dumps(arguments)), - ) + monkeypatch.setattr(policybench, "__path__", [str(cr.ROOT / "policybench")]) + for model in cr.WEB_MODELS: + monkeypatch.setenv( + pilot.provider_key(cr.arm_profile(model, "web").model_id), "synthetic" + ) + if validate: + assert cr.preflight(design, cr.WEB_MODELS) == [] + return w, design -def stateless_model(**kwargs): - """A tool-contract model: one calculation, then its answers. It reads the - conversation so far, so the two lanes can share it.""" - if not any(m.get("role") == "tool" for m in kwargs["messages"]): - arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} - return chat_response([chat_call(pilot.TOOL_NAME, arguments, "c1")]) - outputs = {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} - return chat_response([chat_call("submit_outputs", {"outputs": outputs}, "s1")]) +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +def test_preflight_requires_expected_costs_to_match_the_development_projection( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + if kind == "linked": + design, models = copy.deepcopy(web), cr.WEB_MODELS + path = Path(design["calculator"]["path"]) + changed = json.loads(path.read_text()) + else: + design, models = fresh_stage_design(w, web, tmp_path, kind) + changed = design + for field in ("expected_costs", "reserves"): + changed[field] = {model: value * 2 for model, value in changed[field].items()} + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, models) + assert problems and all( + "development projection gives" in problem for problem in problems + ), "expected costs must match the registered development projection" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("fault", INVALID_SAVED_COSTS) +def test_development_projection_refuses_an_invalid_saved_attempt_cost( + world, tmp_path, monkeypatch, kind, fault +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + folder = "web" if kind == "web" else "pe3" + path = next((Path(design["development_runs"]) / models[0] / folder).rglob("*.json")) + record = json.loads(path.read_text()) + invalidate_saved_cost(record, fault) + path.write_text(json.dumps(record)) + problems = cr.preflight(design, models) + assert len(problems) == 1 and "expected-cost projection" in problems[0], ( + "development projection requires every saved attempt cost to be valid" + ) -def stateless_responses_model(**kwargs): - """A Responses tool-contract model: calculate, then submit the answers.""" - from types import SimpleNamespace +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_requires_the_registered_record_count( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + folder = "web" if kind == "web" else "pe3" + path = next((Path(design["development_runs"]) / models[0] / folder).rglob("*.json")) + path.unlink() + problems = cr.preflight(design, models) + assert ( + len(problems) == 1 + and "development" in problems[0] + and "expected" in problems[0] + ), "development projection requires the registered final-record counts" - if "previous_response_id" not in kwargs: - name, call_id = pilot.TOOL_NAME, "c1" - arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("direction", [-1, 1]) +def test_development_projection_accepts_its_tolerance_and_refuses_the_next_float( + world, tmp_path, monkeypatch, kind, direction +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + if kind == "linked": + design, models = copy.deepcopy(web), cr.WEB_MODELS + path = Path(design["calculator"]["path"]) + changed = json.loads(path.read_text()) + stage = cr.STAGES["calculator"] else: - name, call_id = "submit_outputs", "s1" - arguments = { - "outputs": {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + design, models = fresh_stage_design(w, web, tmp_path, kind) + changed, stage = design, cr.STAGES[kind] + model = stage.models[0] + projected = cr.registered_expected_costs(Path(changed["development_runs"]), stage)[ + model + ] + boundary = projected + direction * 1e-6 + # The greatest representable displacement that stays within the tolerance. + if abs(boundary - projected) > 1e-6: + boundary = math.nextafter(boundary, projected) + outward = math.inf if direction > 0 else -math.inf + for value, accepted in ( + (boundary, True), + (math.nextafter(boundary, outward), False), + ): + changed["expected_costs"][model] = value + total = math.fsum(changed["expected_costs"].values()) + share = cr.WEB_CAP if stage.kind == "web" else cr.CALCULATOR_CAP + changed["caps"] = { + m: share * c / total for m, c in changed["expected_costs"].items() } - return SimpleNamespace( - id=f"resp_{call_id}", - output=[ - SimpleNamespace( - type="function_call", - id=f"fc_{call_id}", - call_id=call_id, - name=name, - arguments=json.dumps(arguments), - status="completed", - ) - ], - output_text=None, - status="completed", - incomplete_details=None, - usage=SimpleNamespace( - input_tokens=1000, - output_tokens=200, - total_tokens=1200, - input_tokens_details=SimpleNamespace(cached_tokens=0), - output_tokens_details=SimpleNamespace(reasoning_tokens=0), - ), - model=kwargs["model"], + changed["reserves"] = { + m: 3 * c / cr.HOUSEHOLDS for m, c in changed["expected_costs"].items() + } + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, models) + assert (problems == []) is accepted, ( + "development projection uses the inclusive one-microdollar tolerance" + ) + if not accepted: + assert len(problems) == 1 and "development projection gives" in problems[0] + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_accepts_the_exact_one_microdollar_tolerance( + tmp_path, kind +): + stage = cr.STAGES[kind] + synthetic_development_tree(tmp_path, kind, {model: 1e-9 for model in stage.models}) + projected = cr.registered_expected_costs(tmp_path, stage) + model = stage.models[0] + boundary = projected[model] + 1e-6 + assert boundary - projected[model] == 1e-6 + design = {"development_runs": str(tmp_path), "expected_costs": dict(projected)} + design["expected_costs"][model] = boundary + assert cr.projection_problems(design, stage) == [], ( + "development projection accepts exactly its one-microdollar tolerance" + ) + design["expected_costs"][model] = math.nextafter(boundary, math.inf) + assert len(cr.projection_problems(design, stage)) == 1, ( + "development projection refuses one float beyond its one-microdollar tolerance" ) -def stateless_json_model(**kwargs): - """A Gemini JSON-contract model: calculate, then answer in JSON text.""" - if not any(m.get("role") == "tool" for m in kwargs["messages"]): - response = stateless_model(**kwargs) - response.model = kwargs["model"] - return response - response = chat_response([]) - response.model = kwargs["model"] - response.choices[0].finish_reason = "stop" - response.choices[0].message.content = json.dumps( - {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} +def test_development_projection_ignores_a_non_directory_unused_arm(tmp_path): + stage = cr.STAGES["calculator"] + synthetic_development_tree( + tmp_path, "calculator", {model: 1.0 for model in stage.models} + ) + before = cr.registered_expected_costs(tmp_path, stage) + (tmp_path / stage.models[0] / "abl-unused").write_text("unused") + raised, after = None, None + try: + after = cr.registered_expected_costs(tmp_path, stage) + except (OSError, ValueError) as error: + raised = error + assert raised is None and after == before, ( + "development projection preserves the report's empty non-directory inventories" ) - return response -@pytest.mark.parametrize( - "model, completion", - [ - ("claude-haiku-5.5", stateless_model), - ("gpt-6-luna", stateless_responses_model), - ("gemini-3.1-pro-preview", stateless_json_model), - ], -) -def test_the_harness_conversation_runs_under_the_runner(world, model, completion): - """Each API and answer contract completes real harness conversations in - both runner lanes, with the registered feature frame for every arm. +def test_web_development_projection_counts_the_other_board_when_one_is_a_file(tmp_path): + stage = cr.STAGES["web"] + synthetic_development_tree(tmp_path, "web", {model: 1.0 for model in stage.models}) + before = cr.registered_expected_costs(tmp_path, stage) + arm = tmp_path / stage.models[0] / "web" + mctest = arm / "mctest" + for path in (mctest / "households").iterdir(): + path.rename(arm / "mafree/households" / f"moved-{path.name}") + (mctest / "households").rmdir() + mctest.rmdir() + mctest.write_text("unused") + raised, after = None, None + try: + after = cr.registered_expected_costs(tmp_path, stage) + except (OSError, ValueError) as error: + raised = error + assert raised is None and after == before, ( + "web development projection preserves the report's " + "empty non-directory inventories" + ) - The confirmatory roster's Geminis use tools. The older board profile - covers the harness's Gemini JSON path in a fake design, without changing - the production roster or any model's answer contract.""" - w = world(models=(model,)) - w.run.converse = cr.converse_with_harness - w.run.completion_for = lambda profile: completion - w.execute() - labels = labels_of(model) - attempts = w.attempts(model) - assert len(attempts) == N_DRAW * len(labels) - assert {x["status"] for x in attempts} == {"complete"} - messages = w.run.lanes[model].pool.messages - assert len(messages) == N_DRAW * len(labels) # one calculation per attempt - frames = [m for m in messages if isinstance(m, tuple)] - assert len(frames) == N_DRAW * (len(labels) - 1) # every arm but pe - sid = w.draw_order[0] - label = "loo-names" if "loo-names" in labels else "pe3" - rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) - profile = cr.arm_profile(model, label) - assert rec["arm"] == cr.arm_string(label) and rec["attempts"] == [] - assert rec["tool_features"] == sorted(profile.tool_features) - assert rec["predictions"] == {v: 0.0 for v in VARIABLES} - assert run_problems(w) == [] +@pytest.mark.parametrize("linked", [False, True]) +def test_web_preflight_spend_declarations_require_nonboolean_numbers( + world, tmp_path, monkeypatch, linked +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + target = calculator if linked else design + original = target["spent_before"] + for value in (str(original), False): + target["spent_before"] = value + if linked: + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.web_preflight(design) + assert any("must be a nonboolean number" in problem for problem in problems), ( + "web floor declarations require nonboolean numbers before conversion" + ) -@pytest.mark.parametrize( - "model, completion, api, contract, label", - [ - (model, completion, api, contract, label) + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_uses_the_registered_mean_and_arm_arithmetic( + tmp_path, kind +): + stage = cr.STAGES[kind] + expected = {model: 2.0 for model in stage.models} + synthetic_development_tree(tmp_path, kind, expected) + model = stage.models[0] + folder = "web" if kind == "web" else "pe3" + path = next((tmp_path / model / folder).rglob("*.json")) + record = json.loads(path.read_text()) + record["totals"]["cost_usd"] += 1.0 + path.write_text(json.dumps(record)) + got = cr.registered_expected_costs(tmp_path, stage) + increment = ( + cr.HOUSEHOLDS + / (100 if kind == "web" else 320) + * (1 if kind == "web" else len(cr.ARMS)) + ) + assert math.isclose(got[model], expected[model] + increment, abs_tol=1e-12), ( + "development projection preserves per-arm means " + "and registered household multipliers" + ) + + +def fresh_stage_design(w, web, tmp_path, kind): + """Keep every unrelated preflight input valid for an isolated budget fault.""" + if kind == "web": + return copy.deepcopy(web), cr.WEB_MODELS + design = copy.deepcopy(w.design) + design["runs"] = str(tmp_path / "fresh-calculator-runs") + return design, cr.STAGES["calculator"].models + + +def test_web_spent_before_covers_calculator_declaration_and_all_saved_costs( + world, tmp_path, monkeypatch +): + w, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["spent_before"] = 6.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + final = next(Path(design["no_tools"]["runs"]).rglob("households/*.json")) + record = json.loads(final.read_text()) + record["totals"]["cost_usd"] = 1.0 + final.write_text(json.dumps(record)) + monkeypatch.setattr(cr, "calculator_evidence", lambda _: [88.0]) + for refused in (89.0, 93.99): + design["spent_before"] = refused + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and "recorded spend $94.0" in problems[0], ( + "web spent_before must cover the calculator declaration plus its costs" + ) + design["spent_before"] = 94.0 + assert cr.preflight(design, cr.WEB_MODELS) == [], ( + "web spent_before accepts the complete $94 declared-and-recorded floor" + ) + calculator["spent_before"] = 0.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + design["spent_before"] = 88.0 + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 2 and "recorded spend $89.0" in problems[1], ( + "web spent_before also covers the complete saved-cost inventory" + ) + assert "linked calculator spent_before" in problems[0], ( + "the linked calculator declaration independently covers saved no-tools costs" + ) + + +def test_linked_calculator_spent_before_covers_its_own_no_tools_floor( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["spent_before"] = 0.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + final = next(Path(design["no_tools"]["runs"]).rglob("households/*.json")) + record = json.loads(final.read_text()) + record["totals"]["cost_usd"] = 1.0 + final.write_text(json.dumps(record)) + monkeypatch.setattr(cr, "calculator_evidence", lambda _: [88.0]) + design["spent_before"] = 89.0 + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and "linked calculator spent_before" in problems[0], ( + "a linked calculator declaration must cover its own saved no-tools floor" + ) + for amount, accepted in ( + (1.0 - 1e-6, True), + (math.nextafter(1.0 - 1e-6, -math.inf), False), + ): + calculator["spent_before"] = amount + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, cr.WEB_MODELS) + assert (problems == []) is accepted, ( + "the linked calculator floor uses the inclusive one-microdollar tolerance" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["expected_costs", "caps", "reserves"]) +def test_design_money_containers_have_exactly_the_registered_model_keys( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + original = ( + json.loads(path.read_text()) if kind == "linked" else copy.deepcopy(design) + ) + for fault in (None, [], "invalid", "extra-good", "extra-bad"): + changed = copy.deepcopy(original) + if isinstance(fault, str) and fault.startswith("extra-"): + changed[field]["unused"] = 1.0 if fault == "extra-good" else "bad" + else: + changed[field] = fault + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + else: + design = changed + problems = cr.preflight(design, models) + prefix = "linked calculator: " if kind == "linked" else "" + assert problems == [ + prefix + f"{field} must be an object with exactly the registered model keys" + ], "money containers must be objects with exactly the registered model keys" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["expected_costs", "caps", "reserves"]) +def test_design_money_values_are_numbers_before_conversion( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + original = ( + json.loads(path.read_text()) if kind == "linked" else copy.deepcopy(design) + ) + model = cr.STAGES["calculator" if kind == "linked" else kind].models[0] + for value in (str(original[field][model]), True): + changed = copy.deepcopy(original) + changed[field][model] = value + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + else: + design = changed + problems = cr.preflight(design, models) + assert ( + len(problems) == 1 + and ( + "finite and positive" + if field == "expected_costs" + else "finite allocation" + ) + in problems[0] + ), "money values must be nonboolean numbers before conversion" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["overall_cap", "spent_before"]) +def test_design_money_scalars_refuse_malformed_values_without_raising( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + original = ( + json.loads(path.read_text()) if kind == "linked" else copy.deepcopy(design) + ) + for value in (10**400, str(original[field]), None, True, math.nan): + changed = copy.deepcopy(original) + changed[field] = value + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + else: + design = changed + raised, problems = None, [] + try: + problems = cr.preflight(design, models) + if kind == "web": + assert cr.web_preflight(design) + except (TypeError, ValueError, OverflowError, AttributeError) as error: + raised = error + assert raised is None and problems, ( + "malformed monetary scalars must produce recorded refusals without raising" + ) + + +@pytest.mark.parametrize("field", ["overall_cap", "spent_before"]) +def test_stage_money_scalars_require_nonboolean_numbers(field): + for value in ("1", True, False): + design = {"overall_cap": 150.0, "spent_before": 1.0} + design[field] = value + assert cr.stage_start_problems(design), ( + "stage monetary scalars require nonboolean numbers before conversion" + ) + + +def test_stage_money_scalars_are_finite_and_prior_spend_is_nonnegative(): + for field, value in ( + ("overall_cap", math.inf), + ("overall_cap", math.nan), + ("spent_before", math.inf), + ("spent_before", math.nan), + ("spent_before", -0.01), + ): + design = {"overall_cap": 150.0, "spent_before": 1.0} + design[field] = value + assert cr.stage_start_problems(design), ( + "stage money must be finite and prior spend must be nonnegative" + ) + + +@pytest.mark.parametrize("check", ["preflight", "web_preflight"]) +def test_malformed_design_objects_become_recorded_problems(check): + for design in (None, [], "invalid", True): + raised, problems = None, [] + try: + problems = ( + cr.preflight(design, cr.WEB_MODELS) + if check == "preflight" + else cr.web_preflight(design) + ) + except (TypeError, ValueError, OverflowError, AttributeError) as error: + raised = error + assert raised is None and len(problems) == 1 and "malformed" in problems[0], ( + "malformed design objects must become recorded problems without raising" + ) + + +def test_malformed_design_paths_become_recorded_problems(world, tmp_path): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + design["preregistration"] = None + raised, problems = None, [] + try: + problems = cr.preflight(design, (cr.CHEAP[0],)) + except (TypeError, ValueError, OverflowError, AttributeError) as error: + raised = error + assert raised is None and len(problems) == 1 and "malformed" in problems[0], ( + "malformed design paths must become recorded problems without raising" + ) + + +def test_calculator_preflight_requires_the_registered_150_dollar_cap( + world, tmp_path, monkeypatch +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, "calculator") + assert cr.preflight(design, models) == [] + for cap in (149.99, 150.01, 270.0): + design["overall_cap"] = cap + assert cr.preflight(design, models) == [ + "the calculator design's overall_cap must be 150" + ], "calculator preflight fixes the whole-run cap at $150" + + +def test_linked_calculator_design_requires_the_registered_150_dollar_cap( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["overall_cap"] = 270.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the linked calculator design's overall_cap must be 150" + ], "the web preflight also fixes its linked calculator cap at $150" + + +def test_linked_calculator_design_refuses_a_start_at_its_175_dollar_ceiling( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + design["spent_before"] = 200.0 + for spent, refused in ((174.99, False), (175.0, True), (175.01, True)): + calculator["spent_before"] = spent + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, cr.WEB_MODELS) == ( + [ + "linked calculator: spent_before must be finite " + "and below the stage ceiling" + ] + if refused + else [] + ), "a linked calculator design must have started below its $175 ceiling" + + +@pytest.mark.parametrize("field", ["expected_costs", "caps", "reserves"]) +def test_linked_calculator_design_requires_registered_allocations( + world, tmp_path, monkeypatch, field +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + model = cr.CHEAP[0] + calculator[field][model] = ( + 0.0 if field == "expected_costs" else calculator[field][model] + 0.1 + ) + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and problems[0].startswith("linked calculator: "), ( + "a linked calculator design must use its registered allocations" + ) + + +@pytest.mark.parametrize("kind,ceiling", [("calculator", 175.0), ("web", 295.0)]) +def test_stage_preflight_refuses_start_at_or_above_the_ceiling( + world, tmp_path, monkeypatch, kind, ceiling +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + design["spent_before"] = ceiling - 0.01 + assert cr.preflight(design, models) == [], "a start below the ceiling is accepted" + for amount in (ceiling, ceiling + 0.01): + design["spent_before"] = amount + assert cr.preflight(design, models) == [ + "spent_before must be finite and below the stage ceiling" + ], "the stage preflight refuses a start at or above its ceiling" + + +@pytest.mark.parametrize("ceiling", [175.0, 295.0]) +def test_ceiling_is_exhausted_at_construction_at_or_above_its_limit(ceiling): + below = cr.Ceiling(ceiling, ceiling - 0.01) + assert not below.exhausted() and below.crossed_at is None + for amount in (ceiling, ceiling + 0.01): + spent = cr.Ceiling(ceiling, amount) + assert spent.exhausted() and spent.crossed_at is not None, ( + "a ceiling reached before construction is already exhausted" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("cost", [None, 0.0, -1.0, math.inf, math.nan, True]) +def test_stage_allocation_requires_each_expected_cost_to_be_finite_and_positive( + world, tmp_path, monkeypatch, kind, cost +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + model = models[0] + if cost is None: + del design["expected_costs"][model] + else: + design["expected_costs"][model] = cost + if isinstance(cost, float) and math.isfinite(cost): + total = sum(design["expected_costs"].values()) + stage_cap = 120.0 if kind == "web" else 145.0 + design["caps"] = { + m: stage_cap * value / total + for m, value in design["expected_costs"].items() + } + design["reserves"] = { + m: 3 * value / cr.HOUSEHOLDS + for m, value in design["expected_costs"].items() + } + assert cr.preflight(design, models) == [ + "expected_costs must be an object with exactly the registered model keys" + if cost is None + else f"{model}: expected_costs must be finite and positive" + ], "each stage model needs a finite positive expected cost" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_requires_caps_to_follow_the_proportional_split( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + model = models[0] + design["caps"][model] += 0.01 + problems = cr.preflight(design, models) + assert len(problems) == 1 and "and the split gives" in problems[0], ( + "each stage cap must follow its registered proportional split" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_requires_three_times_expected_cost_reserves( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + model = models[0] + design["reserves"][model] += 0.001 + problems = cr.preflight(design, models) + assert len(problems) == 1 and "and the rule gives" in problems[0], ( + "each stage reserve is three times its expected cost over 120 households" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_accepts_the_reports_cap_and_reserve_tolerances( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + design["caps"][models[0]] += 0.0049 + design["reserves"][models[0]] += 0.00049 + assert cr.preflight(design, models) == [], ( + "stage allocation checks use the report's $0.005/$0.0005 tolerances" + ) + + +def allocation_guard_design(kind, *, target_cost=1.0): + stage = cr.STAGES[kind] + expected = {model: 1.0 for model in stage.models} + expected[stage.models[0]] = target_cost + return allocated_design(stage, expected) + + +def allocated_design(stage, expected): + total = math.fsum(expected.values()) + share = cr.WEB_CAP if stage.kind == "web" else cr.CALCULATOR_CAP + return { + "kind": stage.kind, + "expected_costs": expected, + "caps": {model: share * cost / total for model, cost in expected.items()}, + "reserves": { + model: 3 * cost / cr.HOUSEHOLDS for model, cost in expected.items() + }, + } + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("field", ["caps", "reserves"]) +@pytest.mark.parametrize("value", [math.nan, math.inf, -math.inf, True, False]) +def test_each_stage_allocation_is_finite_and_nonboolean_alone(kind, field, value): + stage = cr.STAGES[kind] + model = stage.models[0] + design = allocation_guard_design(kind) + if value is True: + if field == "caps": + share = cr.WEB_CAP if kind == "web" else cr.CALCULATOR_CAP + expected = {m: (share - 1) / (len(stage.models) - 1) for m in stage.models} + expected[model] = 1.0 + design = allocated_design(stage, expected) + else: + design = allocation_guard_design(kind, target_cost=cr.HOUSEHOLDS / 3) + elif value is False: + design = allocation_guard_design(kind, target_cost=1e-12) + assert cr.allocation_problems(design, stage) == [] + if isinstance(value, bool): + tolerance = 0.005 if field == "caps" else 0.0005 + assert abs(float(value) - design[field][model]) <= tolerance + design[field][model] = value + assert cr.allocation_problems(design, stage) == [ + f"{model}: {field} needs a finite allocation" + ], "each allocation must be finite and nonboolean even when its formula agrees" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("field", ["caps", "reserves"]) +@pytest.mark.parametrize("side", [-1, 1]) +def test_each_stage_allocation_accepts_exact_tolerance_and_refuses_next_float( + kind, field, side +): + stage = cr.STAGES[kind] + model = stage.models[0] + tolerance = 0.005 if field == "caps" else 0.0005 + allocation = 2 * tolerance if side < 0 else tolerance + if field == "caps": + share = 120 if kind == "web" else 145 + expected = { + m: (share - allocation) / (len(stage.models) - 1) for m in stage.models + } + expected[model] = allocation + design = allocated_design(stage, expected) + else: + design = allocation_guard_design( + kind, target_cost=allocation * cr.HOUSEHOLDS / 3 + ) + want = design[field][model] + assert want == allocation + boundary = want + side * tolerance + assert abs(boundary - want) == tolerance + design[field][model] = boundary + assert cr.allocation_problems(design, stage) == [], ( + "allocation formulas include their exact tolerance on both sides" + ) + design[field][model] = math.nextafter(boundary, math.inf if side > 0 else -math.inf) + problems = cr.allocation_problems(design, stage) + assert len(problems) == 1 and f"{model}: " in problems[0], ( + "allocation formulas refuse one representable step beyond either tolerance" + ) + + +def test_web_declared_calculator_floor_uses_one_flat_fsum(world, tmp_path, monkeypatch): + _, design = complete_web_design(world, tmp_path, monkeypatch) + monkeypatch.setattr(cr, "CEILING_MARGIN", 1e17) + costs, declared_prior = [1e16, 1.0], 1.0 + calculator = json.loads(Path(design["calculator"]["path"]).read_text()) + assert calculator["spent_before"] == declared_prior + assert declared_prior + math.fsum(costs) == 1e16 + assert math.fsum([declared_prior, *costs]) == 1e16 + 2 + monkeypatch.setattr(cr, "calculator_evidence", lambda _: costs) + design["spent_before"] = 1e16 + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and "recorded spend" in problems[0], ( + "the calculator declaration and all calculator costs require one flat fsum" + ) + design["spent_before"] = math.fsum([declared_prior, *costs]) + assert cr.preflight(design, cr.WEB_MODELS) == [] + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_refuses_an_overflowing_expected_cost_total(kind): + stage = cr.STAGES[kind] + design = { + "kind": kind, + "expected_costs": {model: 1e308 for model in stage.models}, + "caps": {model: 20.0 for model in stage.models}, + "reserves": {model: 0.1 for model in stage.models}, + } + assert cr.allocation_problems(design, stage) == [ + "the expected costs do not sum to a finite positive amount" + ], "each stage allocation must refuse an overflowing expected-cost total" + + +def test_web_gemini_preflight_refuses_the_wrong_litellm_alone_before_writes( + world, tmp_path, monkeypatch, capsys +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + real_version = cr.importlib.metadata.version + monkeypatch.setattr( + cr.importlib.metadata, + "version", + lambda name: "9.9.9" if name == "litellm" else real_version(name), + ) + problem = ( + "litellm 9.9.9 is installed; the Gemini search patch was " + f"checked against {pilot.GEMINI_SEARCH_PATCH_LITELLM} only" + ) + assert cr.preflight(design, cr.WEB_MODELS) == [problem], ( + "an otherwise valid Gemini web design must have only the litellm refusal" + ) + path = tmp_path / "web-design.json" + path.write_text(json.dumps(design)) + monkeypatch.setattr(pilot, "_require_engine", lambda board: None) + assert cr.main(["run", str(path)]) == 1 + assert f"The run cannot start:\n- {problem}" in capsys.readouterr().out + assert not Path(design["runs"]).exists(), ( + "the litellm refusal must precede every web startup write" + ) + + +def test_calculator_gemini_preflight_has_no_web_litellm_version_check( + world, tmp_path, monkeypatch +): + w, _ = complete_web_design(world, tmp_path, monkeypatch) + calculator = {**w.design, "runs": str(tmp_path / "fresh-calculator-runs")} + real_version = cr.importlib.metadata.version + monkeypatch.setattr( + cr.importlib.metadata, + "version", + lambda name: "9.9.9" if name == "litellm" else real_version(name), + ) + assert cr.preflight(calculator, ("gemini-3.5-flash-lite",)) == [], ( + "a calculator design must remain valid regardless of the web patch version" + ) + + +def rewritten_chain(path, rows, seed): + """Write synthetic evidence while preserving every intended chain link.""" + texts, heads = [], [seed] + for row in rows: + text = json.dumps({**row, "prev": heads[-1]}) + texts.append(text) + heads.append(cr.line_sha(text)) + path.write_text("\n".join(texts) + "\n") + return texts, heads + + +def rewrite_calculator_anchors(w, anchors): + """Keep pushes and snapshots bound when a test changes an anchor's data.""" + path = w.runs() / "anchors.jsonl" + old_heads = [cr.line_sha(line) for line in path.read_text().splitlines()] + texts, heads = rewritten_chain( + path, anchors, cr.chain_seed(w.design["frozen_sha256"], "", path.name) + ) + positions = {head: i for i, head in enumerate(old_heads)} + pushes = w.runs() / "anchor-pushes.jsonl" + rows = [json.loads(line) for line in pushes.read_text().splitlines()] + for row in rows: + position = positions[row["anchor"]] + row["anchor"] = heads[position + 1] + snapshot = w.runs() / f"anchors.{row['anchor']}.push-snapshot.jsonl" + snapshot.write_text("\n".join(texts[: position + 1]) + "\n") + pushes.write_text("\n".join(json.dumps(row) for row in rows) + "\n") + + +def rewrite_calculator_log(w, model, name, rows): + """Update a changed log's anchor prefixes and final pushed snapshot too.""" + path = w.runs() / model / name + _, heads = rewritten_chain( + path, rows, cr.chain_seed(w.design["frozen_sha256"], model, name) + ) + anchors = [ + json.loads(line) + for line in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + for i, anchor in enumerate(anchors): + count = ( + len(rows) + if i == len(anchors) - 1 + else min(anchor["models"][model][name]["lines"], len(rows)) + ) + anchor["models"][model][name] = {"lines": count, "sha256": heads[count]} + rewrite_calculator_anchors(w, anchors) + + +def rewrite_calculator_record(w, model, label, sid, value, *, archive=None): + """Keep a deliberately malformed payload's surrounding hashes valid.""" + folder = w.folder(model, label) + path = folder / ( + f"{sid}.json" if archive is None else f"attempts/{sid}-{archive}.json" + ) + if isinstance(value, bytes): + path.write_bytes(value) + else: + path.write_text(json.dumps(value)) + rows = w.lines(model) + number = archive or max( + row["attempt"] + for row in rows + if row.get("household") == sid and row.get("arm") == label + ) + for row in rows: + if (row.get("household"), row.get("arm"), row.get("attempt")) == ( + sid, + label, + number, + ): + row["sha256"] = cr.sha256_file(path) + if archive is not None: + final_path = folder / f"{sid}.json" + final = json.loads(final_path.read_text()) + final["attempts"][archive - 1]["sha256"] = cr.sha256_file(path) + final_path.write_text(json.dumps(final)) + for row in rows: + if (row.get("household"), row.get("arm"), row.get("attempt")) == ( + sid, + label, + len(final["attempts"]) + 1, + ): + row["sha256"] = cr.sha256_file(final_path) + rewrite_calculator_log(w, model, "run-log.jsonl", rows) + + +@pytest.mark.parametrize("fault", ["missing", "wrong", "unreadable", "kind"]) +def test_web_preflight_refuses_each_calculator_design_link_fault_alone( + world, tmp_path, monkeypatch, fault +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + if fault == "missing": + del design["calculator"] + elif fault == "wrong": + design["calculator"]["sha256"] = "0" * 64 + elif fault == "unreadable": + design["calculator"]["path"] += ".missing" + else: + path = Path(design["calculator"]["path"]) + prior = json.loads(path.read_text()) + prior["kind"] = "web" + path.write_text(json.dumps(prior)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.web_preflight(design) + assert len(problems) == 1 and "calculator design" in problems[0] + + +@pytest.mark.parametrize("field", cr.SHARED_STAGE_FIELDS + cr.WEB_SCORING_FIELDS) +def test_web_preflight_refuses_each_shared_input_difference_alone( + world, tmp_path, monkeypatch, field +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + prior = json.loads(path.read_text()) + # Change only the linked design's field: the web inputs stay independently valid. + prior[field] = ( + {"different": True} + if field in ("eligible_pool", "no_tools") + else (["different"] if field == "draw_order" else "different") + ) + path.write_text(json.dumps(prior)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert f"web and calculator designs differ in {field}" in cr.web_preflight(design) + + +@pytest.mark.parametrize("root", ["no_tools", "calculator"]) +@pytest.mark.parametrize("direction", ["same", "inside", "parent"]) +def test_web_preflight_refuses_each_overlapping_runs_root_alone( + world, tmp_path, monkeypatch, root, direction +): + w, design = complete_web_design(world, tmp_path, monkeypatch) + other = Path(design["no_tools"]["runs"] if root == "no_tools" else w.design["runs"]) + design["runs"] = str( + other + if direction == "same" + else other / "nested" + if direction == "inside" + else other.parent + ) + assert cr.web_preflight(design) == [ + "the web runs root overlaps the no-tools or calculator runs root" + ] + + +@pytest.mark.parametrize( + "fault,detail", + [ + ("summary", "calculator stage has no run summary"), + ("unresolved", "calculator summary leaves a request unresolved"), + ("in-flight", "calculator attempts remain in flight"), + ("host", "'end'"), + ("chain", "invalid hash chain"), + ("anchor", "calculator stage has no final anchor"), + ("push", "calculator final anchor has no successful final push"), + ("snapshot", "calculator final push snapshot differs from its anchors"), + ("record", "calculator run log differs from attempt records"), + ], +) +def test_web_preflight_refuses_each_unended_calculator_evidence_rule_alone( + world, tmp_path, monkeypatch, fault, detail +): + w, design = complete_web_design(world, tmp_path, monkeypatch) + model = cr.CHEAP[0] + if fault in ("summary", "unresolved"): + path = next((w.runs() / model).glob("run-summary-*.json")) + if fault == "summary": + path.unlink() + else: + summary = json.loads(path.read_text()) + summary["abandoned_running"] = 1 + path.write_text(json.dumps(summary)) + elif fault == "in-flight": + path = w.runs() / model / "in-flight" / "still-running" + path.mkdir(parents=True) + elif fault == "host": + path = w.runs() / "host" / "host.json" + host = json.loads(path.read_text()) + del host["end"] + path.write_text(json.dumps(host)) + elif fault == "chain": + path = w.runs() / model / "run-log.jsonl" + rows = path.read_text().splitlines() + first = json.loads(rows[0]) + first["prev"] = "0" * 64 + rows[0] = json.dumps(first) + path.write_text("\n".join(rows) + "\n") + elif fault == "anchor": + (w.runs() / "anchors.jsonl").write_text("") + elif fault == "push": + path = w.runs() / "anchor-pushes.jsonl" + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[-1]["outcome"] = "error" + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n") + elif fault == "snapshot": + head = cr.line_sha((w.runs() / "anchors.jsonl").read_text().splitlines()[-1]) + (w.runs() / f"anchors.{head}.push-snapshot.jsonl").write_text("altered") + else: + path = w.folder(model, "pe") / f"{w.draw_order[0]}.json" + saved = json.loads(path.read_text()) + saved["totals"]["cost_usd"] += 1 + path.write_text(json.dumps(saved)) + problems = cr.web_preflight(design) + assert len(problems) == 1 and problems[0].startswith( + "the calculator stage has not ended:" + ) + assert detail in problems[0], f"calculator closed-evidence rule: {fault}" + + +@pytest.mark.parametrize( + "fault,detail", + [ + ("summary/object", "expected a JSON object"), + ("summary/model", "calculator summary leaves a request unresolved"), + ("summary/boolean-zero", "calculator summary leaves a request unresolved"), + ("summary/missing-abandoned", "calculator summary leaves a request unresolved"), + ("summary/missing-utc", "'finished_at_utc'"), + ("summary/invalid-utc", "does not match format"), + ("summary/null-utc", "strptime() argument 1 must be str"), + ("summary/attempt-count", "calculator summary or log leaves out attempts"), + ("summary/final-count", "calculator summary or log leaves out attempts"), + ("summary/supervisors", "calculator admitted households have not finished"), + ("host/object", "expected a JSON object"), + ("host/reversed-span", "calculator host log has no closed span"), + ("host/boolean-time", "calculator end evidence holds a nonfinite number"), + ("host/infinite-time", "calculator end evidence holds a nonfinite number"), + ("chain/admission", "invalid hash chain"), + ("chain/host", "invalid hash chain"), + ("chain/anchors", "invalid hash chain"), + ("chain/run-row-object", "invalid hash chain"), + ("run/unsettled", "calculator requests are unresolved or unsettled"), + ("run/late-unresolved", "calculator requests are unresolved or unsettled"), + ("payload/object", "invalid calculator attempt record"), + ("payload/prior-attempts", "invalid calculator attempt record"), + ("payload/attempts-type", "calculator attempts must be a list"), + ("payload/missing-attempts", "'attempts'"), + ("payload/totals-type", "calculator totals must be an object"), + ("payload/missing-cost", "calculator cost must be a present number"), + ("payload/boolean-cost", "calculator cost must be a present number"), + ("payload/string-cost", "calculator cost must be a present number"), + ("payload/negative-cost", "calculator cost must be finite and nonnegative"), + ("payload/nan-cost", "calculator cost must be finite and nonnegative"), + ("payload/infinite-cost", "calculator cost must be finite and nonnegative"), + ("payload/overflow-cost", "int too large to convert to float"), + ("payload/malformed-json", "Expecting property name"), + ("payload/invalid-utf8", "utf-8"), + ("archive-payload/object", "invalid calculator attempt record"), + ("archive-payload/boolean-cost", "calculator cost must be a present number"), + ("link/object", "calculator archive links are invalid"), + ("link/number-gap", "calculator archive links are invalid"), + ("link/boolean-number", "calculator archive links are invalid"), + ("link/missing-file", "calculator archives differ from final links"), + ("link/extra-file", "calculator archives differ from final links"), + ("link/path", "calculator archive hash or path differs"), + ("link/hash", "calculator archive hash or path differs"), + ("link/orphan", "calculator archive has no final"), + ("log/duplicate-attempt", "calculator run log differs from attempt records"), + ("log/unknown-attempt", "calculator run log differs from attempt records"), + ("log/omitted-attempt", "calculator summary or log leaves out attempts"), + ("log/reversed-attempt", "calculator attempt has not ended"), + ("log/boolean-time", "calculator end evidence holds a nonfinite number"), + ("log/nonfinite-time", "calculator end evidence holds a nonfinite number"), + ("late/negative-cost", "calculator late spend is negative"), + ("late/boolean-cost", "calculator end evidence holds a nonfinite number"), + ("late/nonfinite-cost", "calculator end evidence holds a nonfinite number"), + ("manifest/missing", "calculator stage needs exactly one manifest"), + ("manifest/extra", "calculator stage needs exactly one manifest"), + ("manifest/hash", "calculator anchor manifest differs"), + ("anchor/missing-model", "calculator anchor does not cover every model"), + ("anchor/extra-model", "calculator anchor does not cover every model"), + ("anchor/manifest", "calculator anchor manifest differs"), + ("anchor/boolean-count", "calculator anchor differs from its logs"), + ("anchor/outside-count", "calculator anchor differs from its logs"), + ("anchor/head", "calculator anchor differs from its logs"), + ("anchor/stale-run", "calculator final anchor omits log lines"), + ("anchor/stale-admission", "calculator final anchor omits log lines"), + ("anchor/stale-host", "calculator final anchor omits log lines"), + ("push/missing", "calculator final anchor has no successful final push"), + ("push/row-object", "invalid hash chain"), + ("push/returncode", "calculator final anchor has no successful final push"), + ( + "push/boolean-returncode", + "calculator final anchor has no successful final push", + ), + ("push/anchor", "calculator final anchor has no successful final push"), + ("push/reversed-time", "calculator final push has invalid times"), + ("push/before-anchor", "calculator final push has invalid times"), + ("push/boolean-time", "calculator end evidence holds a nonfinite number"), + ("push/nonfinite-time", "calculator end evidence holds a nonfinite number"), + ], +) +def test_web_preflight_refuses_each_closed_calculator_evidence_fault_alone( + world, tmp_path, monkeypatch, fault, detail +): + """Every closure refusal identifies its own fault in otherwise valid evidence. + + Log and anchor mutations retain valid chains and matching push snapshots. + Malformed payloads retain their surrounding record and archive digests, so + a generic downstream hash failure cannot stand in for the named refusal. + """ + group, change = fault.split("/", 1) + w, design = complete_web_design( + world, tmp_path, monkeypatch, archives=group in ("archive-payload", "link") + ) + model, sid = cr.CHEAP[0], w.draw_order[0] + folder = w.runs() / model + record_path = w.folder(model, "pe") / f"{sid}.json" + if group == "summary": + path = next(folder.glob("run-summary-*.json")) + value = json.loads(path.read_text()) + if change == "object": + value = [] + elif change == "model": + value["model"] = "different-model" + elif change == "boolean-zero": + value["abandoned_running"] = False + elif change == "missing-abandoned": + del value["abandoned_running"] + elif change == "missing-utc": + del value["finished_at_utc"] + elif change in ("invalid-utc", "null-utc"): + value["finished_at_utc"] = ( + "invalid UTC" if change == "invalid-utc" else None + ) + elif change in ("attempt-count", "final-count"): + value["attempts" if change == "attempt-count" else "final_records"] += 1 + else: + value["households_finished"].remove(sid) + path.write_text(json.dumps(value)) + elif group == "host": + path = w.runs() / "host" / "host.json" + value = json.loads(path.read_text()) + if change == "object": + value = [] + else: + value["end"] = { + "reversed-span": value["start"] - 1, + "boolean-time": True, + "infinite-time": math.inf, + }[change] + path.write_text(json.dumps(value)) + elif group == "chain": + path = { + "admission": folder / "admission-log.jsonl", + "host": w.runs() / "host" / "host-log.jsonl", + "anchors": w.runs() / "anchors.jsonl", + "run-row-object": folder / "run-log.jsonl", + }[change] + texts = path.read_text().splitlines() + if change == "run-row-object": + texts[0] = "[]" + else: + first = json.loads(texts[0]) + first["prev"] = "0" * 64 + texts[0] = json.dumps(first) + path.write_text("\n".join(texts) + "\n") + elif group in ("run", "log", "late"): + rows = w.lines(model) + position = next(i for i, row in enumerate(rows) if "event" not in row) + row = rows[position] + if group == "run": + if change == "unsettled": + row["class"] = "unsettled" + else: + rows.append( + { + "event": "late-unresolved", + "time": row["end"], + "household": row["household"], + "arm": row["arm"], + "attempt": row["attempt"], + "counted_usd": 0.0, + "outcome": "unresolved-at-ledger-close", + } + ) + elif group == "late": + cost = { + "negative-cost": -0.01, + "boolean-cost": True, + "nonfinite-cost": math.inf, + }[change] + rows.append( + { + "event": "late-cost", + "time": row["end"], + "household": row["household"], + "arm": row["arm"], + "attempt": row["attempt"], + "cost_usd": cost, + } + ) + elif change == "duplicate-attempt": + rows.append(dict(row)) + elif change == "unknown-attempt": + row["attempt"] = 99 + elif change == "omitted-attempt": + del rows[position] + elif change == "reversed-attempt": + row["end"] = row["start"] - 1 + else: + row["end"] = True if change == "boolean-time" else math.inf + if group == "log" and change in ("duplicate-attempt", "omitted-attempt"): + summary_path = next(folder.glob("run-summary-*.json")) + summary = json.loads(summary_path.read_text()) + summary["attempts"] = sum("event" not in item for item in rows) + summary_path.write_text(json.dumps(summary)) + rewrite_calculator_log(w, model, "run-log.jsonl", rows) + elif group in ("payload", "archive-payload"): + archive = 1 if group == "archive-payload" else None + path = ( + record_path + if archive is None + else record_path.parent / "attempts" / f"{sid}-1.json" + ) + value = json.loads(path.read_text()) + if change == "object": + value = [] + elif change == "prior-attempts": + value["prior_attempts"] = [] + elif change == "attempts-type": + value["attempts"] = 1 + elif change == "missing-attempts": + del value["attempts"] + elif change == "totals-type": + value["totals"] = [] + elif change == "missing-cost": + del value["totals"]["cost_usd"] + elif change == "malformed-json": + value = b"{" + elif change == "invalid-utf8": + value = b"\xff" + else: + value["totals"]["cost_usd"] = { + "boolean-cost": True, + "string-cost": "0.01", + "negative-cost": -0.01, + "nan-cost": math.nan, + "infinite-cost": math.inf, + "overflow-cost": 10**400, + }[change] + rewrite_calculator_record(w, model, "pe", sid, value, archive=archive) + elif group == "link": + final = json.loads(record_path.read_text()) + archive_path = record_path.parent / "attempts" / f"{sid}-1.json" + if change == "object": + final["attempts"][0] = None + elif change == "number-gap": + final["attempts"][0]["n"] = 2 + elif change == "boolean-number": + final["attempts"][0]["n"] = True + elif change == "missing-file": + archive_path.unlink() + elif change == "extra-file": + (archive_path.parent / f"{sid}-3.json").write_bytes( + archive_path.read_bytes() + ) + elif change == "orphan": + (archive_path.parent / "orphan-1.json").write_bytes( + archive_path.read_bytes() + ) + else: + final["attempts"][0]["file" if change == "path" else "sha256"] = "different" + if change in ("object", "number-gap", "boolean-number", "path", "hash"): + rewrite_calculator_record(w, model, "pe", sid, final) + elif group == "manifest": + path = next(folder.glob("manifest-*.json")) + if change == "missing": + path.unlink() + elif change == "extra": + path.with_name("manifest-extra.json").write_bytes(path.read_bytes()) + else: + value = json.loads(path.read_text()) + value["synthetic_change"] = True + path.write_text(json.dumps(value)) + elif group == "anchor": + anchors = [ + json.loads(line) + for line in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + final = anchors[-1] + if change == "missing-model": + del final["models"][model] + elif change == "extra-model": + final["models"]["different-model"] = dict(final["models"][model]) + elif change == "manifest": + final["models"][model]["manifest"] = "0" * 64 + elif change in ("boolean-count", "outside-count", "head"): + entry = final["models"][model]["run-log.jsonl"] + if change == "head": + entry["sha256"] = "0" * 64 + else: + entry["lines"] = ( + True if change == "boolean-count" else entry["lines"] + 1 + ) + else: + name = { + "stale-run": "run-log.jsonl", + "stale-admission": "admission-log.jsonl", + "stale-host": "host-log.jsonl", + }[change] + path = w.runs() / ("host" if change == "stale-host" else model) / name + seed = ( + cr.HOST_SEED + if change == "stale-host" + else cr.chain_seed(w.design["frozen_sha256"], model, name) + ) + texts = path.read_text().splitlines() + entry = { + "lines": len(texts) - 1, + "sha256": cr.line_sha(texts[-2]) if len(texts) > 1 else seed, + } + if change == "stale-host": + final[name] = entry + else: + final["models"][model][name] = entry + rewrite_calculator_anchors(w, anchors) + else: + path = w.runs() / "anchor-pushes.jsonl" + pushes = [json.loads(line) for line in path.read_text().splitlines()] + if change == "missing": + pushes = [] + elif change == "row-object": + pushes[-1] = [] + else: + final = pushes[-1] + if change == "returncode": + final["returncode"] = 1 + elif change == "boolean-returncode": + final["returncode"] = False + elif change == "anchor": + final["anchor"] = "0" * 64 + elif change == "reversed-time": + final["ended"] = final["started"] - 1 + elif change == "before-anchor": + final["started"] = final["anchor_time"] - 1 + else: + final["ended"] = True if change == "boolean-time" else math.inf + path.write_text( + "\n".join(json.dumps(row) for row in pushes) + ("\n" if pushes else "") + ) + problems = cr.web_preflight(design) + assert len(problems) == 1, ( + f"one calculator closed-evidence refusal for {fault}: {problems}" + ) + assert problems[0].startswith("the calculator stage has not ended:") + assert detail in problems[0], ( + f"calculator closed-evidence rule {fault}: {problems[0]}" + ) + + +def test_web_preflight_counts_all_attempts_stubs_and_late_spend_in_one_fsum( + world, tmp_path, monkeypatch +): + w, design = complete_web_design(world, tmp_path, monkeypatch) + calculator = json.loads(Path(design["calculator"]["path"]).read_text()) + calculator["spent_before"] = 0.0 + Path(design["calculator"]["path"]).write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(Path(design["calculator"]["path"])) + # A flat sum retains tiny costs that per-model subtotals would round away. + monkeypatch.setattr(cr, "calculator_evidence", lambda _: [1e16, 1.0, 1.0]) + design["spent_before"] = 1e16 + problems = cr.web_preflight(design) + assert len(problems) == 1 and "recorded spend" in problems[0] + design["spent_before"] = math.fsum([1e16, 1.0, 1.0]) + assert cr.web_preflight(design) == [] + + +def test_web_preflight_applies_the_recorded_spend_tolerance( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + total = design["spent_before"] + design["spent_before"] = total - 1e-6 + assert cr.web_preflight(design) == [] + design["spent_before"] = math.nextafter(total - 1e-6, -math.inf) + assert any("recorded spend" in problem for problem in cr.web_preflight(design)) + + +def test_a_calculator_stage_ended_by_a_terminal_stop_can_precede_web(world): + w = world() + model, sid = cr.CHEAP[0], w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): { + "status": "provider_error", + "error": "BadRequestError: synthetic startup defect", + "turns": 0, + } + } + w.execute() + assert w.events(model, "stop") + assert len([row for row in w.attempts(model) if row["household"] == sid]) == 1 + expected = [ + cr.attempt_cost( + json.loads( + (w.folder(m, row["arm"]) / f"{row['household']}.json").read_text() + ) + ) + for m in w.models + for row in w.attempts(m) + ] + assert math.fsum(cr.calculator_evidence(w.design)) == math.fsum(expected) + + +def test_calculator_end_spend_includes_replaced_stubs_and_provider_attempts(world): + w = world() + model, sid = cr.CHEAP[0], w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"raise_after": True}, + (model, first, sid, 2): { + "status": "provider_error", + "error": "RateLimitError: synthetic", + }, + } + w.execute() + rows = w.attempts(model) + assert { + row["class"] for row in rows if row["household"] == sid and row["arm"] == first + } == {"none", "provider", "ok"} + expected_attempts = ( + N_DRAW + * (len(cr.CHEAP) * len(cr.ARMS) + len(cr.FRONTIER) * len(cr.FRONTIER_ARMS)) + + 2 + ) + assert math.fsum(cr.calculator_evidence(w.design)) == pytest.approx( + expected_attempts * COST + ) + + +def test_calculator_end_spend_includes_late_spend_after_a_closed_deadline( + world, monkeypatch +): + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.2) + w = world() + model, sid = cr.CHEAP[0], w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = {(model, first, sid, 1): {"sleep": 0.5, "cost": 0.73}} + w.execute() + late = math.fsum(event["cost_usd"] for event in w.events(model, "late-cost")) + assert late == 0.73 + assert not w.events(model, "late-unresolved") + costs = cr.calculator_evidence(w.design) + assert 0.73 in costs + assert math.fsum(costs) == pytest.approx( + w.run.ceiling.spent_usd - w.design["spent_before"] + ) + + +def test_web_design_refuses_frontier_arms_and_unregistered_web_labels( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + design["frontier_arms"] = {} + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the design's arms are not the registered ones" + ] + del design["frontier_arms"] + design["frontier_models"] = [] + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the design's arms are not the registered ones" + ] + del design["frontier_models"] + design["arms"]["pe"] = design["arms"].pop("web") + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the design's arms are not the registered ones" + ] + + +def test_web_preflight_requires_the_registered_whole_run_cap( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + for cap in (150, 270.01, "270", True, None, math.inf): + design["overall_cap"] = cap + assert cr.web_preflight(design) == ["the web design's overall_cap must be 270"] + + +def test_web_execute_patches_gemini_before_its_first_completion( + world, keep_litellm_gemini_config +): + config = keep_litellm_gemini_config + original = { + name: config.__dict__[name] + for name in ( + "_resolve_search_tool_conflict", + "_drop_search_tools_mixed_with_functions", + ) + } + w = world(kind="web", models=("gemini-3.5-flash-lite",)) + observed = [] + + def completion(**kwargs): + assert all( + config.__dict__[name] is not value for name, value in original.items() + ), "execute must install both Gemini patch methods before completion" + observed.append(True) + + def converse(lanes, sid, label, tool_call, spend): + lanes.completion() + return w.provider(lanes, sid, label, tool_call, spend) + + w.run.completion_for = lambda profile: completion + w.run.converse = converse + w.execute() + assert len(observed) == N_DRAW * len(cr.WEB_ARMS), ( + "the patched Gemini completion must run for every web arm and household" + ) + + +def test_web_stage_lanes_caps_provider_rerun_and_operator_resume( + world, +): + w = world(kind="web", reserve=0.06, caps={m: 0.12 for m in cr.WEB_MODELS}) + sid, model = w.draw_order[0], cr.WEB_MODELS[0] + w.provider.specs = { + (model, "web", sid, 1): { + "status": "provider_error", + "error": "RateLimitError: synthetic transport", + }, + (model, "web+pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: synthetic account", + "turns": 0, + "resume_after": 0.03, + }, + (model, "web+pe3", sid, 2): {"calls": [{"household": {}}]}, + } + w.execute() + assert run_problems(w) == [] + assert w.models == cr.WEB_MODELS + for name in w.models: + lanes = w.run.lanes[name] + assert lanes.labels == cr.WEB_ARMS + assert [lanes.households(i) for i in (1, 2)] == [ + w.draw_order[::2], + w.draw_order[1::2], + ] + assert any( + row["decision"] == "refuse" for row in w.lines(name, "admission-log.jsonl") + ) + assert lanes.pool.closed + assert any( + row["class"] == "provider" and row["action"] == "rerun" + for row in w.attempts(model) + ) + assert [ + event["event"] + for event in w.events(model) + if event["event"] in ("pause", "resume") + ] == ["pause", "resume"] + assert len(w.run.lanes[model].pool.messages) == 1 + + +def test_web_stage_ceiling_covers_the_whole_run_cap_plus_margin_and_prior_spend(world): + w = world(kind="web", overall_cap=270, spent_before=294.98, reserve=0.01).execute() + assert w.run.ceiling.budget_usd == 295.0 + assert w.run.ceiling.exhausted() + assert all(w.events(model, "ceiling") for model in cr.WEB_MODELS) + assert run_problems(w) == [] + + +def test_web_dry_run_prints_its_models_arms_lanes_caps_and_whole_run_ceiling( + world, tmp_path, monkeypatch, capsys +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = tmp_path / "web-design.json" + path.write_text(json.dumps(design)) + monkeypatch.setattr(pilot, "_require_engine", lambda board: None) + assert cr.main(["run", str(path), "--dry-run"]) == 0 + text = capsys.readouterr().out + assert all( + f"{model}: arms web, web+pe3; lanes 3 and 3" in text for model in cr.WEB_MODELS + ) + assert "hard ceiling $295.00" in text + + +@pytest.mark.parametrize( + "turns,truncated,want", + [(0, False, "none"), (1, False, "provider"), (1, True, "model")], +) +def test_web_request_timeout_keeps_calculator_stage_failure_precedence( + turns, truncated, want +): + # Web requests get no internal timeout retry; the record decides the class. + rec = record( + "provider_error", "Timeout: synthetic", turns=turns, truncated=truncated + ) + assert cr.classify(rec, 1, 2, None)[0] == want + + +# ---------------------------------------------------------------- the harness + + +def chat_response(calls): + from types import SimpleNamespace + + message = SimpleNamespace( + role="assistant", content=None, tool_calls=calls, function_call=None + ) + usage = SimpleNamespace( + prompt_tokens=1000, + completion_tokens=200, + total_tokens=1200, + reasoning_tokens=0, + prompt_tokens_details=SimpleNamespace(cached_tokens=0), + completion_tokens_details=None, + ) + return SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="tool_calls")], + usage=usage, + model="claude-haiku-5-5", + ) + + +def chat_call(name, arguments, call_id): + from types import SimpleNamespace + + return SimpleNamespace( + id=call_id, + type="function", + function=SimpleNamespace(name=name, arguments=json.dumps(arguments)), + ) + + +def stateless_model(**kwargs): + """A tool-contract model: one calculation, then its answers. It reads the + conversation so far, so the two lanes can share it.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + return chat_response([chat_call(pilot.TOOL_NAME, arguments, "c1")]) + outputs = {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + return chat_response([chat_call("submit_outputs", {"outputs": outputs}, "s1")]) + + +def stateless_responses_model(**kwargs): + """A Responses tool-contract model: calculate, then submit the answers.""" + from types import SimpleNamespace + + if "previous_response_id" not in kwargs: + name, call_id = pilot.TOOL_NAME, "c1" + arguments = {"situation": json.dumps({"people": {}}), "variables": ["snap"]} + else: + name, call_id = "submit_outputs", "s1" + arguments = { + "outputs": {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + } + return SimpleNamespace( + id=f"resp_{call_id}", + output=[ + SimpleNamespace( + type="function_call", + id=f"fc_{call_id}", + call_id=call_id, + name=name, + arguments=json.dumps(arguments), + status="completed", + ) + ], + output_text=None, + status="completed", + incomplete_details=None, + usage=SimpleNamespace( + input_tokens=1000, + output_tokens=200, + total_tokens=1200, + input_tokens_details=SimpleNamespace(cached_tokens=0), + output_tokens_details=SimpleNamespace(reasoning_tokens=0), + ), + model=kwargs["model"], + ) + + +def stateless_json_model(**kwargs): + """A Gemini JSON-contract model: calculate, then answer in JSON text.""" + if not any(m.get("role") == "tool" for m in kwargs["messages"]): + response = stateless_model(**kwargs) + response.model = kwargs["model"] + return response + response = chat_response([]) + response.model = kwargs["model"] + response.choices[0].finish_reason = "stop" + response.choices[0].message.content = json.dumps( + {v: {"value": 0, "explanation": "value = 0"} for v in VARIABLES} + ) + return response + + +@pytest.mark.parametrize( + "model, completion", + [ + ("claude-haiku-5.5", stateless_model), + ("gpt-6-luna", stateless_responses_model), + ("gemini-3.1-pro-preview", stateless_json_model), + ], +) +def test_the_harness_conversation_runs_under_the_runner(world, model, completion): + """Each API and answer contract completes real harness conversations in + both runner lanes, with the registered feature frame for every arm. + + The confirmatory roster's Geminis use tools. The older board profile + covers the harness's Gemini JSON path in a fake design, without changing + the production roster or any model's answer contract.""" + w = world(models=(model,)) + w.run.converse = cr.converse_with_harness + w.run.completion_for = lambda profile: completion + w.execute() + labels = labels_of(model) + attempts = w.attempts(model) + assert len(attempts) == N_DRAW * len(labels) + assert {x["status"] for x in attempts} == {"complete"} + messages = w.run.lanes[model].pool.messages + assert len(messages) == N_DRAW * len(labels) # one calculation per attempt + frames = [m for m in messages if isinstance(m, tuple)] + assert len(frames) == N_DRAW * (len(labels) - 1) # every arm but pe + sid = w.draw_order[0] + label = "loo-names" if "loo-names" in labels else "pe3" + rec = json.loads((w.folder(model, label) / f"{sid}.json").read_text()) + profile = cr.arm_profile(model, label) + assert rec["arm"] == cr.arm_string(label) and rec["attempts"] == [] + assert rec["tool_features"] == sorted(profile.tool_features) + assert rec["predictions"] == {v: 0.0 for v in VARIABLES} + assert run_problems(w) == [] + + +@pytest.mark.parametrize( + "model, completion, api, contract, label", + [ + (model, completion, api, contract, label) for model, completion, api, contract in [ ("gpt-6-luna", stateless_responses_model, "responses", "tool"), ("gemini-3.1-pro-preview", stateless_json_model, "chat", "json"), @@ -3350,3 +5488,720 @@ def test_the_treatment_code_is_the_treatment_commits(): assert changed == 0, ( "move TREATMENT_COMMIT to the last commit that touches TREATMENT_PATHS" ) + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize( + "field,value", [("schema_hash_exempt", []), ("stated_engine_overrides", {})] +) +def test_preflight_refuses_the_presence_of_each_rehearsal_option( + world, tmp_path, monkeypatch, kind, field, value +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + target = json.loads(path.read_text()) if kind == "linked" else design + target[field] = value + if kind == "linked": + path.write_text(json.dumps(target)) + design["calculator"]["sha256"] = cr.sha256_file(path) + prefix = "linked calculator: " if kind == "linked" else "" + assert cr.preflight(design, models) == [ + prefix + f"the confirmatory design cannot contain the rehearsal option {field}" + ], "preflight refuses rehearsal-option presence even when its value is empty" + + +def test_web_preflight_refuses_a_linked_development_root_difference_alone( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["development_runs"] = str(tmp_path / "other-development-runs") + synthetic_development_tree( + Path(calculator["development_runs"]), "calculator", calculator["expected_costs"] + ) + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, cr.WEB_MODELS) == [ + "web and calculator designs differ in development_runs" + ], "linked calculator and web designs must share their development root" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["caps", "reserves"]) +def test_preflight_refuses_negative_allocations_even_inside_formula_tolerances( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + target = json.loads(path.read_text()) if kind == "linked" else design + stage = cr.STAGES["calculator" if kind == "linked" else kind] + model = stage.models[0] + expected = {m: 1.0 for m in stage.models} + expected[model] = 0.0 + synthetic_development_tree(Path(target["development_runs"]), stage.kind, expected) + expected[model] = 1e-20 + target["expected_costs"] = expected + total = math.fsum(expected.values()) + share = 120 if stage.kind == "web" else 145 + target["caps"] = {m: share * c / total for m, c in expected.items()} + target["reserves"] = {m: 3 * c / cr.HOUSEHOLDS for m, c in expected.items()} + prefix = "linked calculator: " if kind == "linked" else "" + for value, want in ( + (0.0, []), + (-0.0, []), + (-0.0001, [prefix + f"{model}: {field} needs a nonnegative allocation"]), + ): + target[field][model] = value + if kind == "linked": + path.write_text(json.dumps(target)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, models) == want, ( + "allocations must be nonnegative even inside their formula tolerance; " + "signed zero is allowed" + ) + + +@pytest.mark.parametrize("command", ["run", "resume"]) +@pytest.mark.parametrize( + "fault", ["missing", "invalid-json", "null", "list", "no-kind", "bad-kind"] +) +def test_command_line_reports_unreadable_or_nonobject_designs( + tmp_path, capsys, command, fault +): + path = tmp_path / "design.json" + if fault != "missing": + path.write_text( + { + "invalid-json": "{", + "null": "null", + "list": "[]", + "no-kind": "{}", + "bad-kind": '{"kind": "unregistered"}', + }[fault] + ) + argv = [command, str(path)] + (["gpt-6-luna"] if command == "resume" else []) + raised = None + try: + result = cr.main(argv) + except Exception as error: + raised = error + result = None + output = capsys.readouterr().out + prefix = "No resume requested" if command == "resume" else "The run cannot start" + reason = { + "missing": f"[Errno 2] No such file or directory: {str(path)!r}", + "invalid-json": ( + "Expecting property name enclosed in double quotes: " + "line 1 column 2 (char 1)" + ), + "null": "the design must be an object", + "list": "the design must be an object", + "no-kind": "the design's kind must be calculator or web", + "bad-kind": "the design's kind must be calculator or web", + }[fault] + if fault in ("no-kind", "bad-kind"): + assert raised is None and result == 1 and output == f"{prefix}: {reason}\n", ( + "command line handles stage selection inside the startup guard rule" + ) + assert raised is None and result == 1 and output == f"{prefix}: {reason}\n", ( + "the command line reports unreadable JSON and non-object designs " + "as startup refusals" + ) + + +@pytest.mark.parametrize("board", ["missing", None]) +def test_command_line_prints_preflight_refusals_before_reading_the_board( + tmp_path, capsys, monkeypatch, board +): + path = tmp_path / "design.json" + design = {"kind": "calculator"} + if board is None: + design["board"] = None + path.write_text(json.dumps(design)) + monkeypatch.setattr( + pilot, + "_require_engine", + lambda _: pytest.fail("engine check must follow a successful preflight"), + ) + raised = None + try: + result = cr.main(["run", str(path)]) + except Exception as error: + raised = error + result = None + assert ( + raised is None + and result == 1 + and "The run cannot start:\n- the design has no" in capsys.readouterr().out + ), "the command line prints preflight refusals before dependent board reads" + + +@pytest.mark.parametrize("runs", ["missing", None, "", [], {}]) +def test_command_line_resume_refuses_a_design_without_usable_runs( + tmp_path, capsys, runs +): + path = tmp_path / "design.json" + design = {"kind": "calculator"} + if runs != "missing": + design["runs"] = runs + path.write_text(json.dumps(design)) + raised = None + try: + result = cr.main(["resume", str(path), "gpt-6-luna"]) + except Exception as error: + raised = error + result = None + assert ( + raised is None + and result == 1 + and capsys.readouterr().out + == "No resume requested: the design must name a usable runs folder\n" + ), "resume reports missing or unusable runs folders as startup refusals" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_refuses_a_fifo_without_blocking(tmp_path, kind): + stage = cr.STAGES[kind] + synthetic_development_tree(tmp_path, kind, {model: 1.0 for model in stage.models}) + arm = "web" if kind == "web" else "pe3" + path = next((tmp_path / stage.models[0] / arm).rglob("*.json")) + path.unlink() + os.mkfifo(path) + done = threading.Event() + outcome = [] + + def read_projection(): + try: + outcome.append(cr.registered_expected_costs(tmp_path, stage)) + except Exception as error: + outcome.append(error) + finally: + done.set() + + reader = threading.Thread(target=read_projection, daemon=True) + reader.start() + finished = done.wait(2) + if not finished: + # Release an ordinary blocking open in a reverted reader, so even a + # failing guard test leaves no stuck reader or pytest process behind. + try: + descriptor = os.open(path, os.O_WRONLY | os.O_NONBLOCK) + except OSError: + descriptor = None + if descriptor is not None: + try: + os.write(descriptor, b'{"totals":{"cost_usd":0}}') + finally: + os.close(descriptor) + reader.join(2) + assert ( + finished + and len(outcome) == 1 + and isinstance(outcome[0], ValueError) + and str(path) in str(outcome[0]) + and "FIFO" in str(outcome[0]) + ), "development record reads refuse FIFO inputs promptly with a path-bearing error" + + +@pytest.mark.parametrize( + "inventory,count,linked", + [ + (inventory, count, linked) + for inventory, counts in (("cheap", (319, 321)), ("frontier", (9, 11))) + for count in counts + for linked in (False, True) + ] + + [ + (inventory, count, False) + for inventory in ("web", "web-pe3") + for count in (99, 101) + ], + ids=[ + f"{inventory}-{count}-{'linked' if linked else 'direct'}" + for inventory, counts in (("cheap", (319, 321)), ("frontier", (9, 11))) + for count in counts + for linked in (False, True) + ] + + [ + f"{inventory}-{count}-direct" + for inventory in ("web", "web-pe3") + for count in (99, 101) + ], +) +def test_projection_refuses_missing_and_extra_finals_for_each_inventory( + world, tmp_path, monkeypatch, inventory, count, linked +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + kind = "web" if inventory.startswith("web") else "calculator" + design, models = fresh_stage_design(w, web, tmp_path, "web" if linked else kind) + if inventory == "cheap": + model, arm, required = cr.CHEAP[0], "pe3", 320 + elif inventory == "frontier": + model, arm, required = cr.FRONTIER[0], "pe", 10 + else: + model, arm, required = cr.WEB_MODELS[0], inventory, 100 + folder = Path(design["development_runs"]) / model / arm / "mctest/households" + if count < required: + next(folder.glob("*.json")).unlink() + else: + source = next(folder.glob("*.json")) + (folder / "extra-final.json").write_bytes(source.read_bytes()) + problems = cr.preflight(design, models) + assert ( + len(problems) == 1 + and "expected-cost projection" in problems[0] + and f"expected {required}" in problems[0] + and str(count) in problems[0] + and (not linked or problems[0].startswith("linked calculator: ")) + ), ( + "development projection requires exact cheap, frontier and per-web-arm " + "final counts, directly and through the link" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +def test_projection_ignores_unregistered_paths(world, tmp_path, monkeypatch, kind): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + root = Path(design["development_runs"]) + decoys = [ + root / cr.CHEAP[0] / "ablX/mctest/households/extra.json", + root / cr.FRONTIER[0] / "abl-unregistered/mctest/households/extra.json", + root / cr.CHEAP[0] / "pe3/unregistered-board/households/extra.json", + root / cr.CHEAP[0] / "pe3/mctest/households/not-json.txt", + root / cr.CHEAP[0] / "pe3/mctest/households/attempts/extra.json", + root / cr.WEB_MODELS[0] / "web/unregistered-board/households/extra.json", + root / cr.WEB_MODELS[0] / "web/mctest/households/not-json.txt", + root / cr.WEB_MODELS[0] / "web/mctest/households/attempts/extra.json", + root / cr.WEB_MODELS[0] / "unregistered-arm/mctest/households/extra.json", + ] + for path in decoys: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"totals": {"cost_usd": 1e12}})) + assert cr.preflight(design, models) == [], ( + "development projection ignores populated unregistered arms, boards, " + "non-JSON files and nested attempts" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +def test_projection_preserves_registered_summation_and_pooling( + world, tmp_path, monkeypatch, kind +): + # A changed multiplier must reach this test's independent arithmetic oracle. + w, web = complete_web_design(world, tmp_path, monkeypatch, validate=False) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + link = Path(web["calculator"]["path"]) + target = json.loads(link.read_text()) if kind == "linked" else design + root = Path(target["development_runs"]) + + def replace_inventory(model, arms): + import shutil + + for existing in (root / model).iterdir(): + if (kind == "web" and existing.name in ("web", "web-pe3")) or ( + kind != "web" + and (existing.name in ("pe", "pe3") or existing.name.startswith("abl-")) + ): + shutil.rmtree(existing) + for arm, costs in arms: + # Both boards have nonuniform costs in each web arm. + for n, cost in enumerate(costs): + board = "mafree" if kind == "web" and n >= 50 else "mctest" + path = root / model / arm / board / "households" / f"{n:03}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"totals": {"cost_usd": cost}})) + + if kind == "web": + model = cr.WEB_MODELS[0] + costs = [1e16, 1.0] + [1e-16] * 98 + assert sum(costs) == 1e16 and math.fsum(costs) == 1e16 + 2 + replace_inventory( + model, + [("web", costs), ("web-pe3", list(range(1, 101)))], + ) + # Tiny positive costs move fsum across a rounding tie, including on + # Python 3.12, whose ordinary sum already compensates simple losses. + expected = {model: math.fsum([(1e16 + 2) / 100, 5050 / 100]) * 120} + else: + cheap, frontier = cr.CHEAP[0], cr.FRONTIER[0] + costs = [1e16, 1.0] + [1e-16] * 318 + assert sum(costs) == 1e16 and math.fsum(costs) == 1e16 + 2 + replace_inventory( + cheap, + [("abl-alpha", costs[:1]), ("abl-beta", costs[1:3]), ("pe3", costs[3:])], + ) + replace_inventory(frontier, [("pe", list(range(1, 11)))]) + # The registered calculator arithmetic uses ordinary sum and a pooled + # record mean despite unequal arm sizes of 1, 2 and 317 (320 total). + expected = {cheap: 1e16 / 320 * 120 * 7, frontier: 55 / 10 * 120 * 2} + # world uses six households to keep fake end-to-end stages small. Assert + # the real registered projection with its literal 120 households first. + stage = cr.STAGES["web" if kind == "web" else "calculator"] + fake_households = cr.HOUSEHOLDS + monkeypatch.setattr(cr, "HOUSEHOLDS", 120) + try: + got = cr.registered_expected_costs(root, stage) + finally: + monkeypatch.setattr(cr, "HOUSEHOLDS", fake_households) + assert {model: got[model] for model in expected} == expected, ( + "development projections preserve registered sum, fsum, pooled means " + "and literal 120, 7 and 2 multipliers" + ) + target["expected_costs"].update( + {model: cost / 120 * fake_households for model, cost in expected.items()} + ) + total = math.fsum(target["expected_costs"].values()) + share = 120 if kind == "web" else 145 + target["caps"] = {m: share * c / total for m, c in target["expected_costs"].items()} + target["reserves"] = { + m: 3 * c / cr.HOUSEHOLDS for m, c in target["expected_costs"].items() + } + if kind == "linked": + link.write_text(json.dumps(target)) + design["calculator"]["sha256"] = cr.sha256_file(link) + assert cr.preflight(design, models) == [], ( + "development projections preserve registered sum, fsum, pooled means " + "and literal 120, 7 and 2 multipliers" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize( + "origin", + [ + prefix + "PolicyEngine/policybench-v2-anchors" + suffix + for prefix in ( + "https://github.com/", + "git@github.com:", + "ssh://git@github.com/", + ) + for suffix in ("", ".git") + ], +) +def test_anchor_clone_accepts_registered_branch_and_origin(tmp_path, kind, origin): + repo = synthetic_anchor_clone(tmp_path, kind) + subprocess.run( + ["git", "-C", str(repo), "remote", "set-url", "origin", origin], check=True + ) + command = [sys.executable, "anchor.py", "--repo", str(repo)] + assert cr.anchor_clone_problems(command, cr.STAGES[kind]) == [], ( + "anchor clone registered branch and SSH or HTTPS origin rule" + ) + command[-2:] = [f"--repo={repo}"] + assert cr.anchor_clone_problems(command, cr.STAGES[kind]) == [], ( + "anchor clone registered branch and SSH or HTTPS origin rule" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize( + "fault", + [ + "branch", + "detached", + "origin", + "no-origin", + "no-repo", + "two-repos", + "missing-value", + ], +) +def test_anchor_clone_refuses_unregistered_metadata(tmp_path, kind, fault): + repo = synthetic_anchor_clone(tmp_path, kind) + command = [sys.executable, "anchor.py", "--repo", str(repo)] + expected_ref = { + "calculator": "refs/heads/anchors/v2-confirmatory", + "web": "refs/heads/anchors/v2-confirmatory-web", + }[kind] + if fault == "branch": + subprocess.run( + ["git", "-C", str(repo), "checkout", "--quiet", "-b", "unregistered"], + check=True, + ) + elif fault == "detached": + subprocess.run( + ["git", "-C", str(repo), "checkout", "--quiet", "--detach"], check=True + ) + elif fault == "origin": + subprocess.run( + [ + "git", + "-C", + str(repo), + "remote", + "set-url", + "origin", + "https://github.com/example/unregistered.git", + ], + check=True, + ) + elif fault == "no-origin": + subprocess.run( + ["git", "-C", str(repo), "remote", "remove", "origin"], check=True + ) + elif fault == "no-repo": + command = command[:2] + elif fault == "two-repos": + command += [f"--repo={repo}"] + else: + command.pop() + if fault in ("no-repo", "two-repos", "missing-value"): + expected = "the anchor command must name exactly one --repo clone" + rule = "anchor command names exactly one repo clone rule" + elif fault in ("branch", "detached"): + expected = f"the anchor clone {repo}: checked-out branch must be {expected_ref}" + rule = "anchor clone registered checked-out stage branch rule" + else: + expected = ( + f"the anchor clone {repo}: origin must name " + "PolicyEngine/policybench-v2-anchors over SSH or HTTPS" + ) + rule = "anchor clone registered origin rule" + raised, problems = None, None + try: + problems = cr.anchor_clone_problems(command, cr.STAGES[kind]) + except (IndexError, OSError) as error: + raised = error + assert raised is None and problems == [expected], rule + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("fault", ["bare", "unborn"]) +def test_anchor_clone_requires_a_working_tree_and_commit(tmp_path, kind, fault): + repo = tmp_path / "anchor" + branch = { + "calculator": "anchors/v2-confirmatory", + "web": "anchors/v2-confirmatory-web", + }[kind] + if fault == "bare": + source = synthetic_anchor_clone(tmp_path, kind) + subprocess.run( + ["git", "clone", "--quiet", "--bare", str(source), str(repo)], check=True + ) + else: + subprocess.run( + [ + "git", + "init", + "--quiet", + "--template=", + f"--initial-branch={branch}", + str(repo), + ], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(repo), + "config", + "remote.origin.url", + "https://github.com/PolicyEngine/policybench-v2-anchors.git", + ], + check=True, + ) + assert cr.anchor_clone_problems( + ["anchor", "--repo", str(repo)], cr.STAGES[kind] + ) == [f"the anchor clone {repo}: needs a working tree with a checked-out commit"], ( + "anchor clone requires a working tree and checked-out commit rule" + ) + + +def test_anchor_clone_git_read_failure_is_a_refusal(tmp_path, monkeypatch): + repo = tmp_path / "clone" + + def denied(*args, **kwargs): + raise PermissionError("synthetic anchor metadata denial") + + monkeypatch.setattr(cr.subprocess, "run", denied) + raised, problems = None, None + try: + problems = cr.anchor_clone_problems( + ["anchor", "--repo", str(repo)], cr.STAGES["web"] + ) + except OSError as error: + raised = error + assert raised is None and problems == [ + f"the anchor clone {repo}: cannot read Git metadata: " + "synthetic anchor metadata denial" + ], "anchor clone Git read errors become recorded refusals rule" + + +def test_preflight_checks_registered_anchor_clone(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + sentinel = "synthetic registered anchor clone refusal" + seen = [] + + def refused(command, stage): + seen.append((command, stage.kind)) + return [sentinel] + + monkeypatch.setattr(cr, "anchor_clone_problems", refused) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert ( + seen == [(design["anchor_command"], "calculator")] and sentinel in problems + ), "preflight applies registered anchor clone checks rule" + + +@pytest.mark.parametrize("style", ["lf", "crlf", "spaces"]) +def test_run_manifests_hash_the_design_files_exact_bytes( + world, tmp_path, monkeypatch, style +): + """Exercise the real CLI reader and manifest writer with a fake provider.""" + w = world(models=(cr.FRONTIER[0],)) + text = json.dumps(w.design, indent=2) + if style == "crlf": + text = text.replace("\n", "\r\n") + elif style == "spaces": + text = " \n" + text + "\n " + data = text.encode("utf-8") + path = tmp_path / "cli-design.json" + path.write_bytes(data) + reads = [] + real_read = Path.read_bytes + + def counted_read(source): + if source == path: + reads.append(source) + return real_read(source) + + def plan(design, *, design_sha256): + assert design == w.design + w.run.design_sha256 = design_sha256 + return w.run + + monkeypatch.setattr(Path, "read_bytes", counted_read) + monkeypatch.setattr(cr, "preflight", lambda *args: []) + monkeypatch.setattr(pilot, "_require_engine", lambda _: None) + monkeypatch.setattr(cr, "Run", plan) + assert cr.main(["run", str(path)]) == 0 + manifests = list(w.runs().glob("*/manifest-*.json")) + expected = hashlib.sha256(data).hexdigest() + assert manifests and all( + json.loads(manifest.read_text())["design_sha256"] == expected + for manifest in manifests + ), "every manifest binds the exact design file bytes rule" + assert reads == [path], "the runner reads the design file bytes once rule" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_preflight_binds_every_registered_restart_manifest( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, _ = fresh_stage_design(w, web, tmp_path, kind) + stage = cr.STAGES[kind] + models = (stage.models[0],) + data = json.dumps(design).encode("utf-8") + path = tmp_path / "restart-design.json" + path.write_bytes(data) + expected = hashlib.sha256(data).hexdigest() + assert cr.preflight(design, models, expected) == [], "bound restart control rule" + # Inspect every registered folder even when the requested models are a subset. + manifest = Path(design["runs"]) / stage.models[-1] / "manifest-prior.json" + manifest.parent.mkdir(parents=True) + manifest.write_text(json.dumps({"design_sha256": expected})) + decoy_model = cr.FRONTIER[0] if kind == "web" else "claude-sonnet-5.5" + decoy = Path(design["runs"]) / decoy_model / "manifest-decoy.json" + decoy.parent.mkdir(parents=True) + decoy.write_text('{"design_sha256": "unregistered"}') + assert cr.preflight(design, models, expected) == [], "bound restart control rule" + refusal = f"{manifest}: design_sha256 is missing or differs from the design file" + for value in ({}, {"design_sha256": "0" * 64}, []): + manifest.write_text(json.dumps(value)) + assert cr.preflight(design, models, expected) == [refusal], ( + "restart manifests bind every registered model " + "to the same design bytes rule" + ) + manifest.write_text(json.dumps({"design_sha256": expected})) + path.write_bytes(data + b" \n") + changed = hashlib.sha256(path.read_bytes()).hexdigest() + assert cr.preflight(design, models, changed) == [refusal], ( + "restart manifests bind every registered model to the same design bytes rule" + ) + assert cr.preflight(design, models) == [refusal], ( + "restart manifests require an explicit expected design digest rule" + ) + manifest.write_text("{") + problems = cr.preflight(design, models, expected) + assert len(problems) == 1 and problems[0].startswith( + f"{manifest}: cannot read restart manifest: " + ), "malformed restart manifests become recorded refusals rule" + + +@pytest.mark.parametrize( + "kind,model", + [ + ("calculator", "gpt-6-luuna"), + ("calculator", "../x"), + ("calculator", "claude-sonnet-5.5"), + ("web", "gpt-6-sol"), + ], +) +def test_resume_refuses_models_outside_its_registered_stage( + tmp_path, capsys, kind, model +): + design = tmp_path / "design.json" + design.write_text(json.dumps({"kind": kind, "runs": str(tmp_path / "runs")})) + before = set(tmp_path.rglob("*")) + result = cr.main(["resume", str(design), model]) + assert result == 1 and capsys.readouterr().out == ( + f"No resume requested: model {model!r} is not in the {kind} stage\n" + ), "resume accepts only registered stage models before writing rule" + assert set(tmp_path.rglob("*")) == before, ( + "resume accepts only registered stage models before writing rule" + ) + + +def test_resume_reports_write_failures_as_no_resume_requested( + tmp_path, capsys, monkeypatch +): + design = tmp_path / "design.json" + design.write_text( + json.dumps({"kind": "calculator", "runs": str(tmp_path / "runs")}) + ) + + def denied(*args, **kwargs): + raise PermissionError("synthetic resume write denial") + + monkeypatch.setattr(cr, "write_json", denied) + assert cr.main(["resume", str(design), "gpt-6-luna"]) == 1 + assert ( + capsys.readouterr().out + == "No resume requested: synthetic resume write denial\n" + ), "every resume write refusal uses the no-resume prefix rule" + assert not (tmp_path / "runs").exists() + + +def test_preflight_catch_all_keeps_earlier_problems( + world, tmp_path, monkeypatch, capsys +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + design.update(overall_cap=149.0, board=None) + path = tmp_path / "malformed-design.json" + path.write_text(json.dumps(design)) + monkeypatch.setattr( + pilot, "_require_engine", lambda _: pytest.fail("preflight must refuse first") + ) + assert cr.main(["run", str(path)]) == 1 + output = capsys.readouterr().out + early = "- the calculator design's overall_cap must be 150\n" + late = "- the design is malformed: " + assert ( + early in output and late in output and output.index(early) < output.index(late) + ), "preflight catch-all retains earlier problems before malformed design rule"