From f1cbacf603356e8cd3367ffd0cfe29eaa622168d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 23:11:38 -0500 Subject: [PATCH 01/31] Run calculator and web stages from registered stage data. Register each stage's models, per-model arms and folder keys, profile rule and order stream. Require design kind calculator or web before a run or operator resume starts. Preserve the calculator stage's six models, labels, streams, records and failure rules. Run Luna, Flash-Lite and Sonnet concurrently with two lanes each and web versus web+pe3 orders from order/web/. Require web search in both web profiles, prohibit a calculator in web, require every calculator feature in web+pe3, and retain development's benchmark-search prohibition. Use each model's single engine pool for calculator calls and refuse calculator calls in web. Apply the pinned Gemini search compatibility patch and record actual web request timeouts, search tools, benchmark-search rules, timeout retry settings and provider search prices in the manifests. Require the web design's hashed calculator-design link, its calculator kind, and equality of the frozen inputs, engine, draw, treatment, runner commit, workers and no-tools design. Preserve scoring metadata equality for build, exclusions, baseline exclusions, supplementary exclusions, disputed cells, adjudication, adjudicator, judge and published sources. Require overall_cap 270, registered web arms with distinct folders, and absent frontier arms and models. Refuse overlapping web, no-tools and calculator roots. Require closed calculator host evidence, ended model summaries, empty in-flight inventories, settled requests, matching admissions and ended household supervisors, complete hashed finals and sequential archives, complete log chains, matching manifests and final anchors, and a successful final push with its exact immutable snapshot. Allow a terminally stopped calculator stage once its requests settle. Count every prior model's finals, replacements, paid stubs and late spend in one math.fsum and refuse spent_before below that total less one microdollar. Keep admission, reserves, reruns, operator pauses, deadlines, host evidence and the overall cap plus 25 dollar ceiling unchanged. Add synthetic tests for each stage rule, profile refusal, stream, design link, shared field, root overlap, end-evidence refusal, exact spend boundary, paid and late attempts, web lanes, admission, provider reruns, operator resume, dry runs and the ceiling. Check web timeout classifications against the existing failure precedence. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 517 +++++++++++++++++++++++++++++++-- tests/test_confirmatory_run.py | 486 ++++++++++++++++++++++++++++++- 2 files changed, 971 insertions(+), 32 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index d0957ef7..b32451ba 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1,4 +1,4 @@ -"""The confirmatory calculator-interface run. +"""The confirmatory calculator-interface and web-search stages. Usage: python scripts/confirmatory_run.py run DESIGN [--workers N] [--dry-run] @@ -7,13 +7,20 @@ Runs the frozen pre-registration's "Run procedure" and writes its "Run records", which the confirmatory report checks before it scores anything. DESIGN is the JSON file written at the freeze, the file the report reads. +Its required ``kind`` selects the registered calculator or web stage. The +calculator stage keeps the six models and their original arm labels. The web +stage compares ``web`` and ``web+pe3`` on Luna, Flash-Lite and Sonnet, after +the hashed calculator design's stage has closed. Its frozen inputs and scoring +metadata agree with that design, and its prior spend includes both earlier +stages. Both web profiles prohibit benchmark searches, as in development. - **Lanes.** Each model has two lanes. Households alternate between them in draw order, the first drawn to lane 1. A lane runs one household's arms one after another, in the order drawn for that household and model (the stream - ``order/``), then its reruns in passes (second attempts before + ``order/`` for calculator, ``order/web/`` for web), then its + reruns in passes (second attempts before third, each pass in the household's order), and only then decides its next - household. The six models run at once. + household. A stage's models run at once: six calculator models or three web models. - **One engine pool per model**, started once, with no features of its own. Each call carries its arm's features in a call frame (``tool_call_frame``), the message a pool started with those features would send. ``setup-start`` @@ -28,8 +35,9 @@ households has ended a pass whose reruns are undecided, so that whether it has finished is known. - **Spend.** No conversation is stopped for spend, except at the hard - ceiling: the overall cap plus $25, over the spend before the calculator - stage and every model's spend since. At the crossing, no attempt starts, + ceiling: the overall cap plus $25, over the spend before the selected + stage and every model's spend since. The web design's whole-run cap is $270. + At the crossing, no attempt starts, the conversations under way end ``budget_stop`` at their next turn, and every model's run log gets a ``ceiling`` event at the crossing's time. The decision to admit a household and the crossing share one lock, so no @@ -151,6 +159,78 @@ "loo-labels-flat", ) FRONTIER_ARMS = ("pe", "pe3") +WEB_MODELS = ("gpt-6-luna", "gemini-3.5-flash-lite", "claude-sonnet-5.5") +WEB_ARMS = ("web", "web+pe3") + + +@dataclass(frozen=True) +class Stage: + """A registered stage's models, labels, folders, stream and profile rule.""" + + kind: str + models: tuple[str, ...] + model_arms: dict[str, tuple[str, ...]] + folder_keys: dict[str, str] + order_stream: str + profile_rule: str + calculator_label: str + + +STAGES = { + "calculator": Stage( + "calculator", + CHEAP + FRONTIER, + {m: ARMS if m in CHEAP else FRONTIER_ARMS for m in CHEAP + FRONTIER}, + {m: "arms" if m in CHEAP else "frontier_arms" for m in CHEAP + FRONTIER}, + "order/{model}", + "calculator", + "pe3", + ), + "web": Stage( + "web", + WEB_MODELS, + {m: WEB_ARMS for m in WEB_MODELS}, + {m: "arms" for m in WEB_MODELS}, + "order/web/{model}", + "web", + "web+pe3", + ), +} + +# The web stage uses precisely the calculator stage's frozen inputs and settings. +SHARED_STAGE_FIELDS = ( + "preregistration", + "frozen_sha256", + "board", + "engine", + "eligible_pool", + "draw_order", + "treatment_harness", + "run_commit", + "workers", + "no_tools", +) +WEB_SCORING_FIELDS = ( + "build", + "exclusions_sha256", + "baseline_exclusions", + "supplementary_exclusions", + "disputed", + "adjudication", + "adjudicator_model", + "judge", + "published_sources", +) + + +def stage_for(design: dict) -> Stage: + """Refuse a missing or unregistered kind before any stage starts.""" + kind = design.get("kind") + if not isinstance(kind, str) or kind not in STAGES: + raise ValueError("the design's kind must be calculator or web") + return STAGES[kind] + + # The features each label drops from pe3; None for the tool v1 arm. DROPPED = { "pe": None, @@ -237,6 +317,7 @@ EARLY = frozenset({"operator", "harness", "ceiling", "none"}) REQUIRED = ( + "kind", "preregistration", "frozen_sha256", "runs", @@ -246,7 +327,6 @@ "eligible_pool", "draw_order", "arms", - "frontier_arms", "treatment_harness", "run_commit", "caps", @@ -281,11 +361,16 @@ def draw(master: int, pool_ids: list[str], n: int | None = None) -> list[str]: def orders( - master: int, model: str, draw_order: list[str], arms: tuple[str, ...] + master: int, + model: str, + draw_order: list[str], + arms: tuple[str, ...], + *, + kind: str = "calculator", ) -> dict[str, list[str]]: """Each drawn household's arm order for a model, drawn for every household in draw order, so no order depends on which households run.""" - generator = stream(master, f"order/{model}") + generator = stream(master, STAGES[kind].order_stream.format(model=model)) return { sid: [str(a) for a in generator.permutation(list(arms))] for sid in draw_order } @@ -296,6 +381,8 @@ def orders( def arm_string(label: str) -> str: """The harness's arm string for a design label (`TOOL_FEATURES` order).""" + if label in WEB_ARMS: + return label dropped = DROPPED[label] if dropped is None: return "pe" @@ -305,8 +392,36 @@ def arm_string(label: str) -> str: def arm_profile(model: str, label: str) -> pilot.ModelProfile: - """The treatment harness's profile for a model and design label; refused - unless its calculator features are the ones the label registers.""" + """The pinned profile, checked by its stage's registered treatment rule. + + Web profiles retain development's benchmark-search prohibition: the pinned + harness applies it to both web arms and includes it in their notices. + """ + stage = STAGES["web" if label in WEB_ARMS else "calculator"] + if stage.profile_rule == "web": + profile = pilot.model_profile( + model, + arm=label, + prompt_version=PROMPT_VERSION, + benchmark_search_rule=True, + ) + if not profile.uses_web: + raise ValueError(f"{model}/{label}: the harness gives no web search") + if label == "web" and profile.uses_calculator: + raise ValueError(f"{model}/{label}: the harness gives a calculator") + if label == "web+pe3" and ( + not profile.uses_calculator + or set(profile.tool_features) != set(pilot.TOOL_FEATURES) + ): + raise ValueError( + f"{model}/{label}: the harness lacks the full calculator features" + ) + if not profile.benchmark_search_rule: + raise ValueError( + f"{model}/{label}: the harness omits the benchmark-search rule" + ) + pilot.web_search_tool(profile) + return profile profile = pilot.model_profile( model, arm=arm_string(label), prompt_version=PROMPT_VERSION ) @@ -973,7 +1088,9 @@ def __init__(self, run: "Run", model: str, labels: tuple[str, ...], folders: dic self.profiles = {label: arm_profile(model, label) for label in labels} self.cap = float(design["caps"][model]) self.reserve = float(design["reserves"][model]) - self.orders = orders(run.master, model, run.draw_order, labels) + self.orders = orders( + run.master, model, run.draw_order, labels, kind=run.stage.kind + ) self.cond = threading.Condition() self._log_lock = threading.Lock() self.state: dict[str, str] = {} # household -> running, settling, finished @@ -1230,6 +1347,8 @@ def converse(self, sid: str, label: str, n: int) -> tuple[dict, bool]: spend = AttemptSpend(self.run.ceiling) def tool_call(arguments): + if not profile.uses_calculator: + raise RuntimeError(f"{self.model}/{label}: no calculator is offered") message = ( pilot.tool_call_frame(features, arguments) if features else arguments ) @@ -1490,14 +1609,19 @@ def lane(self, lane: int) -> None: def manifest(self) -> dict: board, design = self.run.board, self.run.design - pe3 = arm_profile(self.model, "pe3") + pe3 = self.profiles[self.run.stage.calculator_label] first = sorted(board["scenarios"])[0] def timeout(profile): built = pilot.build_initial_request( board["scenarios"][first], board["variables"][first], profile ) - return built[1].get("timeout") + value = built[1].get("timeout") + return ( + max(float(value or 0), pilot.WEB_REQUEST_TIMEOUT_SECONDS) + if profile.uses_web + else value + ) board_path = Path(design["board"]) return { @@ -1538,6 +1662,18 @@ def timeout(profile): **pilot.profile_treatment(profile), "tool_version": profile.tool_version, "request_timeout_seconds": timeout(profile), + **( + { + "web_search_tool": pilot.web_search_tool(profile), + "benchmark_search_rule": profile.benchmark_search_rule, + "web_timeouts_retried": False, + "search_usd_per_1k": pilot.SEARCH_USD_PER_1K[ + profile.provider + ], + } + if self.run.stage.kind == "web" + else {} + ), } for label, profile in self.profiles.items() }, @@ -1613,7 +1749,7 @@ def default_completion(profile: pilot.ModelProfile): class Run: - """The six models' lanes, the host log and the hard ceiling.""" + """The chosen stage's models and lanes, host log and whole-run ceiling.""" def __init__( self, @@ -1627,6 +1763,7 @@ def __init__( out=sys.stdout, ): self.design, self.out = design, out + self.stage = stage_for(design) self.workers = int(design["workers"]) if workers is None else workers self.converse, self.pool_factory = converse, pool_factory self.completion_for = completion_for @@ -1642,13 +1779,13 @@ def __init__( self.ceiling = Ceiling( float(design["overall_cap"]) + CEILING_MARGIN, float(design["spent_before"]) ) - self.models = tuple(models or CHEAP + FRONTIER) + self.models = tuple(models or self.stage.models) self.lanes = { model: Lanes( self, model, - ARMS if model in CHEAP else FRONTIER_ARMS, - design["arms"] if model in CHEAP else design["frontier_arms"], + self.stage.model_arms[model], + design[self.stage.folder_keys[model]], ) for model in self.models } @@ -1928,7 +2065,10 @@ def execute(self) -> None: setup_open.add(lanes.model) for model, lanes in self.lanes.items(): self.watch(anchor=False) - lanes.completion = self.completion_for(lanes.profiles["pe3"]) + representative = lanes.profiles[self.stage.calculator_label] + if representative.uses_web and representative.provider == "gemini": + pilot.allow_gemini_search_with_functions() + lanes.completion = self.completion_for(representative) stamp = time.strftime("%Y-%m-%dT%H%M%SZ", time.gmtime()) lanes.manifest_sha256 = write_json( lanes.folder / f"manifest-{stamp}.json", lanes.manifest(), indent=2 @@ -2163,11 +2303,329 @@ def host_power() -> tuple[bool, bool]: return "'AC Power'" in batt, '"AppleClamshellState" = No' in lid +def calculator_evidence(design: dict) -> list[float]: + """Require a closed calculator stage and return every recorded cost. + + The existing runner's end evidence is a closed host span, a summary for + every model, empty in-flight inventories, complete attempt records, and + the final anchor covering all current logs and manifests with a successful + final push. Unresolved requests and unsettled attempts prevent closure. + A stopped stage may be closed; its confirmatory standing is a report rule. + """ + from confirmatory_no_tools import ( + read_regular, + saved_children, + saved_json_paths, + ) + + root, frozen = Path(design["runs"]), design["frozen_sha256"] + + def document(path): + value = json.loads(read_regular(path).decode("utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path}: expected a JSON object") + return value + + def attempt_record(path): + # Calculator archives preserve the harness's original bytes, whose + # first attempt need not carry an attempts field. The final adds its + # complete links; the no-tools reader's stronger schema is different. + data = read_regular(path) + record = json.loads(data.decode("utf-8")) + if not isinstance(record, dict) or "prior_attempts" in record: + raise ValueError(f"{path}: invalid calculator attempt record") + if "attempts" in record and not isinstance(record["attempts"], list): + raise ValueError(f"{path}: calculator attempts must be a list") + totals = record.get("totals") + if not isinstance(totals, dict): + raise ValueError(f"{path}: calculator totals must be an object") + cost = totals.get("cost_usd") + if isinstance(cost, bool) or not isinstance(cost, (int, float)): + raise ValueError(f"{path}: calculator cost must be a present number") + if not math.isfinite(float(cost)) or cost < 0: + raise ValueError(f"{path}: calculator cost must be finite and nonnegative") + return record, sha256_bytes(data) + + def lines(path, seed=None): + texts = read_regular(path).decode("utf-8").splitlines() + rows, heads = [], [seed] + for text in texts: + row = json.loads(text) + if not isinstance(row, dict) or ( + seed is not None and row.get("prev") != heads[-1] + ): + raise ValueError(f"{path}: invalid hash chain") + rows.append(row) + heads.append(line_sha(text)) + return rows, heads + + def finite(value): + if ( + isinstance(value, bool) + or not isinstance(value, (float, int)) + or not math.isfinite(value) + ): + raise ValueError("calculator end evidence holds a nonfinite number") + return float(value) + + host = document(root / "host" / "host.json") + if finite(host["end"]) < finite(host["start"]): + raise ValueError("calculator host log has no closed span") + _, host_heads = lines(root / "host" / "host-log.jsonl", HOST_SEED) + anchors_path = root / "anchors.jsonl" + anchors, anchor_heads = lines(anchors_path, chain_seed(frozen, "", "anchors.jsonl")) + if not anchors: + raise ValueError("calculator stage has no final anchor") + costs, heads_by_model = [], {} + for model in STAGES["calculator"].models: + folder = root / model + if saved_children(folder / "in-flight"): + raise ValueError(f"{model}: calculator attempts remain in flight") + summaries = sorted( + p + for p in saved_children(folder) + if p.name.startswith("run-summary-") and p.suffix == ".json" + ) + if not summaries: + raise ValueError(f"{model}: calculator stage has no run summary") + summary = document(summaries[-1]) + if ( + summary.get("model") != model + or type(summary.get("abandoned_running")) is not int + or summary["abandoned_running"] != 0 + ): + raise ValueError(f"{model}: calculator summary leaves a request unresolved") + datetime.strptime(summary["finished_at_utc"], "%Y-%m-%dT%H:%M:%SZ") + run_rows, run_heads = lines( + folder / "run-log.jsonl", chain_seed(frozen, model, "run-log.jsonl") + ) + admissions, admission_heads = lines( + folder / "admission-log.jsonl", + chain_seed(frozen, model, "admission-log.jsonl"), + ) + if any( + row.get("event") == "late-unresolved" or row.get("class") == "unsettled" + for row in run_rows + ): + raise ValueError( + f"{model}: calculator requests are unresolved or unsettled" + ) + if { + row["household"] for row in admissions if row.get("decision") == "admit" + } != set(summary.get("households_finished", [])): + raise ValueError( + f"{model}: calculator admitted households have not finished" + ) + attempts = [row for row in run_rows if "event" not in row] + inventory = {} + model_costs, final_count = [], 0 + for label in STAGES["calculator"].model_arms[model]: + key = STAGES["calculator"].folder_keys[model] + records = folder / design[key][label] / design["stage"] / "households" + for path in saved_json_paths(records): + final_count += 1 + record, digest = attempt_record(path) + links = record["attempts"] + if any( + not isinstance(link, dict) or link.get("n") != n + for n, link in enumerate(links, 1) + ): + raise ValueError(f"{path}: calculator archive links are invalid") + inventory[(path.stem, label, len(links) + 1)] = (digest, record) + archives = { + p.name: p + for p in saved_json_paths(records / "attempts") + if p.stem.rsplit("-", 1)[0] == path.stem + } + expected = {f"{path.stem}-{n}.json" for n in range(1, len(links) + 1)} + if set(archives) != expected: + raise ValueError( + f"{path}: calculator archives differ from final links" + ) + for n, link in enumerate(links, 1): + archive = archives[f"{path.stem}-{n}.json"] + prior, prior_digest = attempt_record(archive) + if ( + link.get("file") != f"attempts/{archive.name}" + or link.get("sha256") != prior_digest + ): + raise ValueError( + f"{path}: calculator archive hash or path differs" + ) + inventory[(path.stem, label, n)] = (prior_digest, prior) + # An archive without a final must not hide a request or its spend. + finals = {p.stem for p in saved_json_paths(records)} + for archive in saved_json_paths(records / "attempts"): + if archive.stem.rsplit("-", 1)[0] not in finals: + raise ValueError(f"{archive}: calculator archive has no final") + seen = set() + for row in attempts: + key = (row["household"], row["arm"], row["attempt"]) + if ( + key in seen + or key not in inventory + or inventory[key][0] != row.get("sha256") + ): + raise ValueError( + f"{model}: calculator run log differs from attempt records" + ) + seen.add(key) + model_costs.append(attempt_cost(inventory[key][1])) + if finite(row["end"]) < finite(row["start"]): + raise ValueError(f"{model}: calculator attempt has not ended") + if ( + seen != set(inventory) + or summary.get("attempts") != len(attempts) + or summary.get("final_records") != final_count + ): + raise ValueError(f"{model}: calculator summary or log leaves out attempts") + costs.extend(model_costs) + for row in run_rows: + if row.get("event") == "late-cost": + late = finite(row["cost_usd"]) + if late < 0: + raise ValueError(f"{model}: calculator late spend is negative") + costs.append(late) + manifests = sorted( + p + for p in saved_children(folder) + if p.name.startswith("manifest-") and p.suffix == ".json" + ) + if len(manifests) != 1: + raise ValueError(f"{model}: calculator stage needs exactly one manifest") + heads_by_model[model] = { + "run-log.jsonl": run_heads, + "admission-log.jsonl": admission_heads, + "manifest": sha256_bytes(read_regular(manifests[0])), + } + for i, anchor in enumerate(anchors): + if set(anchor.get("models", {})) != set(heads_by_model): + raise ValueError("calculator anchor does not cover every model") + covered = [(anchor["host-log.jsonl"], host_heads)] + for model, heads in heads_by_model.items(): + if anchor["models"][model].get("manifest") != heads["manifest"]: + raise ValueError(f"{model}: calculator anchor manifest differs") + covered.extend( + (anchor["models"][model][name], heads[name]) + for name in ("run-log.jsonl", "admission-log.jsonl") + ) + for got, heads in covered: + count = got.get("lines") + if ( + type(count) is not int + or not 0 <= count < len(heads) + or got.get("sha256") != heads[count] + ): + raise ValueError("calculator anchor differs from its logs") + if i == len(anchors) - 1 and count != len(heads) - 1: + raise ValueError("calculator final anchor omits log lines") + final_head = anchor_heads[-1] + pushes, _ = lines(root / "anchor-pushes.jsonl") + if ( + not pushes + or pushes[-1].get("anchor") != final_head + or pushes[-1].get("outcome") != "success" + or pushes[-1].get("returncode") != 0 + ): + raise ValueError("calculator final anchor has no successful final push") + if finite(pushes[-1]["ended"]) < finite(pushes[-1]["started"]) or finite( + pushes[-1]["started"] + ) < finite(anchors[-1]["time"]): + raise ValueError("calculator final push has invalid times") + snapshot = root / f"anchors.{final_head}.push-snapshot.jsonl" + if read_regular(snapshot) != read_regular(anchors_path): + raise ValueError("calculator final push snapshot differs from its anchors") + return costs + + +def web_preflight(design: dict) -> list[str]: + """The web stage follows its hashed, closed calculator stage and all spend.""" + from confirmatory_no_tools import ( + component, + exact_spend, + history, + listed_is_dir, + read_regular, + saved_children, + ) + + problems = [] + if design.get("overall_cap") != 270: + problems.append("the web design's overall_cap must be 270") + try: + link = design["calculator"] + path = Path(link["path"]) + data = read_regular(path) + if sha256_bytes(data) != link["sha256"]: + return [*problems, "the calculator design does not hash as recorded"] + calculator = json.loads(data.decode("utf-8")) + if not isinstance(calculator, dict) or calculator.get("kind") != "calculator": + return [*problems, "the linked calculator design's kind must be calculator"] + except (OSError, ValueError, KeyError, TypeError, RecursionError) as error: + return [*problems, f"the calculator design cannot be read: {error}"] + for field in SHARED_STAGE_FIELDS: + if field not in calculator or design.get(field) != calculator[field]: + problems.append(f"web and calculator designs differ in {field}") + for field in WEB_SCORING_FIELDS: + if design.get(field) != calculator.get(field): + problems.append(f"web and calculator designs differ in {field}") + try: + roots = [ + Path(design["runs"]).resolve(), + Path(design["no_tools"]["runs"]).resolve(), + Path(calculator["runs"]).resolve(), + ] + if any( + roots[0].is_relative_to(other) or other.is_relative_to(roots[0]) + for other in roots[1:] + ): + problems.append( + "the web runs root overlaps the no-tools or calculator runs root" + ) + except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: + problems.append(f"the web runs roots are invalid: {error}") + try: + costs = calculator_evidence(calculator) + spec = design["no_tools"] + for model in saved_children(Path(spec["runs"])): + if listed_is_dir(model): + _, saved = history( + model + / component(spec["folder"]) + / component(spec["stage"]) + / "households" + ) + costs.extend(saved) + spent, declared = exact_spend(costs), float(design["spent_before"]) + if not 0 <= declared < math.inf or declared < spent - 1e-6: + problems.append( + f"web spent_before ${declared} is below no-tools and calculator " + f"recorded spend ${spent}" + ) + except ( + OSError, + ValueError, + KeyError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + ) as error: + problems.append(f"the calculator stage has not ended: {error}") + return problems + + def preflight(design: dict, models: tuple[str, ...]) -> list[str]: """What stops the run from starting.""" missing = [k for k in REQUIRED if k not in design] if missing: return [f"the design has no {', '.join(missing)}"] + try: + stage = stage_for(design) + except ValueError as error: + return [str(error)] + if stage.kind == "calculator" and "frontier_arms" not in design: + return ["the design has no frontier_arms"] problems = [] from confirmatory_no_tools import ( BOARD_INPUT_ERRORS, @@ -2180,13 +2638,15 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: ) problems.extend(stage_problems(design)) + if stage.kind == "web": + problems.extend(web_preflight(design)) try: no_tools_root = Path(design["no_tools"]["runs"]).resolve() calculator_root = Path(design["runs"]).resolve() if no_tools_root.is_relative_to( calculator_root ) or calculator_root.is_relative_to(no_tools_root): - problems.append("the no-tools and calculator runs roots overlap") + problems.append(f"the no-tools and {stage.kind} runs roots overlap") except (KeyError, TypeError, OSError, ValueError, RuntimeError) as error: problems.append(f"the no-tools runs root is invalid: {error}") command = design["anchor_command"] @@ -2277,11 +2737,15 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: installed = pilot._versions()["policyengine-us"] if installed != engine: problems.append(f"policyengine-us {installed} is installed, not {engine}") - if set(design["arms"]) != set(ARMS) or set(design["frontier_arms"]) != set( - FRONTIER_ARMS + mappings = { + stage.folder_keys[model]: stage.model_arms[model] for model in stage.models + } + if any(set(design[key]) != set(labels) for key, labels in mappings.items()) or ( + stage.kind == "web" + and any(key in design for key in ("frontier_arms", "frontier_models")) ): problems.append("the design's arms are not the registered ones") - for name in ("arms", "frontier_arms"): + for name in mappings: if len(set(design[name].values())) != len(design[name]): problems.append(f"two {name} share a folder") @@ -2293,7 +2757,7 @@ def household_paths(stage): for model in models: if model not in design["caps"] or model not in design["reserves"]: problems.append(f"{model}: no cap or reserve") - for label in ARMS if model in CHEAP else FRONTIER_ARMS: + for label in stage.model_arms[model]: try: arm_profile(model, label) except (ValueError, SystemExit) as error: @@ -2346,6 +2810,11 @@ def main(argv=None) -> int: resume.add_argument("--note", default="the account was fixed") args = parser.parse_args(argv) design = json.loads(args.design.read_text()) + try: + stage = stage_for(design) + except ValueError as error: + print(f"The run cannot start: {error}") + return 1 if args.command == "resume": folder = Path(design["runs"]) / args.model write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) @@ -2357,7 +2826,7 @@ def main(argv=None) -> int: f"runs {design.get('workers')} engine workers per model." ) return 1 - models = CHEAP + FRONTIER + models = stage.models problems = preflight(design, models) from confirmatory_no_tools import BOARD_INPUT_ERRORS diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 81861839..e8e0dfb1 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -128,6 +128,99 @@ def test_an_arm_whose_features_are_not_registered_is_refused(monkeypatch): cr.arm_profile("gpt-6-luna", "loo-flat") +def test_stage_table_registers_the_calculator_and_web_models_arms_and_folders(): + calculator, web = cr.STAGES["calculator"], cr.STAGES["web"] + assert calculator.models == cr.CHEAP + cr.FRONTIER + assert calculator.model_arms == { + m: cr.ARMS if m in cr.CHEAP else cr.FRONTIER_ARMS for m in calculator.models + } + assert calculator.folder_keys == { + m: "arms" if m in cr.CHEAP else "frontier_arms" for m in calculator.models + } + assert calculator.order_stream == "order/{model}" + assert calculator.profile_rule == "calculator" + assert web.models == ("gpt-6-luna", "gemini-3.5-flash-lite", "claude-sonnet-5.5") + assert web.model_arms == {m: ("web", "web+pe3") for m in web.models} + assert web.folder_keys == {m: "arms" for m in web.models} + assert web.order_stream == "order/web/{model}" + assert web.profile_rule == "web" + + +@pytest.mark.parametrize("kind", [None, "", "unknown", [], 7]) +def test_design_kind_is_required_and_registered_before_any_stage_starts(kind): + design = {} if kind is None else {"kind": kind} + with pytest.raises(ValueError, match="kind must be calculator or web"): + cr.stage_for(design) + with pytest.raises(ValueError, match="kind must be calculator or web"): + cr.Run(design) + + +@pytest.mark.parametrize("label", cr.WEB_ARMS) +@pytest.mark.parametrize("model", cr.WEB_MODELS) +def test_web_arm_profile_keeps_search_the_benchmark_rule_and_its_calculator( + label, model +): + profile = cr.arm_profile(model, label) + assert profile.arm == label and profile.uses_web + assert profile.benchmark_search_rule is True + assert profile.prompt_version == cr.PROMPT_VERSION + assert profile.uses_calculator == (label == "web+pe3") + assert set(profile.tool_features) == ( + set(pilot.TOOL_FEATURES) if profile.uses_calculator else set() + ) + + +@pytest.mark.parametrize( + "label,changed,message", + [ + ("web", {"uses_web": False}, "no web search"), + ("web+pe3", {"uses_web": False}, "no web search"), + ("web", {"uses_calculator": True}, "gives a calculator"), + ("web+pe3", {"uses_calculator": False}, "full calculator features"), + ("web", {"benchmark_search_rule": False}, "benchmark-search rule"), + ] + + [ + ( + "web+pe3", + {"tool_features": set(pilot.TOOL_FEATURES) - {feature}}, + "full calculator features", + ) + for feature in pilot.TOOL_FEATURES + ], +) +def test_web_arm_profile_refuses_each_wrong_profile_alone( + monkeypatch, label, changed, message +): + from types import SimpleNamespace + + original = cr.arm_profile("gpt-6-luna", label) + values = { + name: getattr(original, name) + for name in ( + "uses_web", + "uses_calculator", + "tool_features", + "benchmark_search_rule", + ) + } + monkeypatch.setattr( + pilot, "model_profile", lambda *a, **k: SimpleNamespace(**{**values, **changed}) + ) + with pytest.raises(ValueError, match=message): + cr.arm_profile("gpt-6-luna", label) + + +def test_web_orders_use_the_web_stream_for_every_drawn_household(): + got = cr.orders(9, cr.WEB_MODELS[0], POOL, cr.WEB_ARMS, kind="web") + generator = cr.stream(9, "order/web/gpt-6-luna") + assert got == { + sid: [str(a) for a in generator.permutation(list(cr.WEB_ARMS))] for sid in POOL + } + assert cr.orders(9, cr.WEB_MODELS[0], POOL[:3], cr.WEB_ARMS, kind="web") == { + sid: got[sid] for sid in POOL[:3] + } + + def litellm_names(groups) -> set[str]: """The names of litellm's exception classes under any of `groups`.""" import inspect @@ -766,6 +859,7 @@ def __init__( no_output=(), models=None, pool=FakePool, + kind="calculator", ): self.tmp = tmp prereg = tmp / "prereg.md" @@ -795,9 +889,11 @@ def __init__( monkeypatch.setitem( pilot._STATED_ENGINE, "version", pilot._STATED_ENGINE["version"] ) - self.models = tuple(models or cr.CHEAP + cr.FRONTIER) + self.kind = kind + self.models = tuple(models or cr.STAGES[kind].models) budget_models = tuple(dict.fromkeys(cr.CHEAP + cr.FRONTIER + self.models)) self.design = { + "kind": kind, "preregistration": str(prereg), "frozen_sha256": frozen, "runs": str(tmp / "runs"), @@ -814,6 +910,9 @@ def __init__( "workers": 2, "anchor_command": [sys.executable, str(ANCHOR_SCRIPT), str(tmp)], } + if kind == "web": + self.design["arms"] = {a: a.replace("+", "-") for a in cr.WEB_ARMS} + del self.design["frontier_arms"] self.readers = Readers() self.provider = FakeProvider(self, specs) self.run = cr.Run( @@ -852,9 +951,7 @@ def events(self, model, kind=None) -> list[dict]: ] def folder(self, model, label) -> Path: - arms = ( - self.design["arms"] if model in cr.CHEAP else self.design["frontier_arms"] - ) + arms = self.design[cr.STAGES[self.kind].folder_keys[model]] return self.runs() / model / arms[label] / "confirm" / "households" @@ -971,8 +1068,8 @@ def run_problems(world: World) -> list[str]: all_attempts = [] ceiling_events = {} for model in world.models: - labels = labels_of(model) - drawn = cr.orders(master, model, world.draw_order, labels) + labels = cr.STAGES[world.kind].model_arms[model] + drawn = cr.orders(master, model, world.draw_order, labels, kind=world.kind) lines = world.attempts(model) events = world.events(model) for name in ("run-log.jsonl", "admission-log.jsonl"): @@ -2071,7 +2168,9 @@ def before_rerun(lanes): def test_the_worker_count_is_the_design_s(tmp_path, capsys): design = tmp_path / "design.json" - design.write_text(json.dumps({"runs": str(tmp_path / "runs"), "workers": 2})) + design.write_text( + json.dumps({"kind": "calculator", "runs": str(tmp_path / "runs"), "workers": 2}) + ) assert cr.main(["run", str(design), "--workers", "3"]) == 1 assert "the design runs 2 engine workers" in capsys.readouterr().out @@ -2372,12 +2471,383 @@ def test_preflight_refuses_overlapping_no_tools_and_calculator_roots( def test_resume_writes_the_operator_s_note(tmp_path): design = tmp_path / "design.json" - design.write_text(json.dumps({"runs": str(tmp_path / "runs")})) + design.write_text( + json.dumps({"kind": "calculator", "runs": str(tmp_path / "runs")}) + ) assert cr.main(["resume", str(design), "gpt-6-luna", "--note", "key rotated"]) == 0 note = json.loads((tmp_path / "runs" / "gpt-6-luna" / "RESUME.json").read_text()) assert note["note"] == "key rotated" +def complete_web_design(world, tmp_path, monkeypatch): + """A web preflight with a genuinely ended synthetic calculator stage.""" + from types import SimpleNamespace + + w = world() + w.design.update(complete_preflight_design(w, tmp_path)) + w.execute() + calculator = tmp_path / "calculator-design.json" + calculator.write_text(json.dumps(w.design)) + design = copy.deepcopy(w.design) + design.update( + kind="web", + runs=str(tmp_path / "web-runs"), + arms={a: a.replace("+", "-") for a in cr.WEB_ARMS}, + calculator={"path": str(calculator), "sha256": cr.sha256_file(calculator)}, + caps={m: 40.0 for m in cr.WEB_MODELS}, + reserves={m: 0.06 for m in cr.WEB_MODELS}, + overall_cap=270.0, + spent_before=math.fsum(cr.calculator_evidence(w.design)), + ) + del design["frontier_arms"] + monkeypatch.setattr( + pilot, "_git", lambda *args: "0" * 40 if args[0] == "rev-parse" else "" + ) + real = subprocess.run + + def git_pins(args, *a, **kw): + if args and args[0] == "git": + return SimpleNamespace(returncode=0, stdout=cr.TREATMENT_COMMIT + "\n") + return real(args, *a, **kw) + + monkeypatch.setattr(subprocess, "run", git_pins) + import policybench + + monkeypatch.setattr(policybench, "__path__", [str(cr.ROOT / "policybench")]) + for model in cr.WEB_MODELS: + monkeypatch.setenv( + pilot.provider_key(cr.arm_profile(model, "web").model_id), "synthetic" + ) + assert cr.preflight(design, cr.WEB_MODELS) == [] + return w, design + + +@pytest.mark.parametrize("fault", ["missing", "wrong", "unreadable", "kind"]) +def test_web_preflight_refuses_each_calculator_design_link_fault_alone( + world, tmp_path, monkeypatch, fault +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + if fault == "missing": + del design["calculator"] + elif fault == "wrong": + design["calculator"]["sha256"] = "0" * 64 + elif fault == "unreadable": + design["calculator"]["path"] += ".missing" + else: + path = Path(design["calculator"]["path"]) + prior = json.loads(path.read_text()) + prior["kind"] = "web" + path.write_text(json.dumps(prior)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.web_preflight(design) + assert len(problems) == 1 and "calculator design" in problems[0] + + +@pytest.mark.parametrize("field", cr.SHARED_STAGE_FIELDS + cr.WEB_SCORING_FIELDS) +def test_web_preflight_refuses_each_shared_input_difference_alone( + world, tmp_path, monkeypatch, field +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + prior = json.loads(path.read_text()) + # Change only the linked design's field: the web inputs stay independently valid. + prior[field] = ( + {"different": True} + if field in ("eligible_pool", "no_tools") + else (["different"] if field == "draw_order" else "different") + ) + path.write_text(json.dumps(prior)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert f"web and calculator designs differ in {field}" in cr.web_preflight(design) + + +@pytest.mark.parametrize("root", ["no_tools", "calculator"]) +@pytest.mark.parametrize("direction", ["same", "inside", "parent"]) +def test_web_preflight_refuses_each_overlapping_runs_root_alone( + world, tmp_path, monkeypatch, root, direction +): + w, design = complete_web_design(world, tmp_path, monkeypatch) + other = Path(design["no_tools"]["runs"] if root == "no_tools" else w.design["runs"]) + design["runs"] = str( + other + if direction == "same" + else other / "nested" + if direction == "inside" + else other.parent + ) + assert cr.web_preflight(design) == [ + "the web runs root overlaps the no-tools or calculator runs root" + ] + + +@pytest.mark.parametrize( + "fault", + [ + "summary", + "unresolved", + "in-flight", + "host", + "chain", + "anchor", + "push", + "snapshot", + "record", + ], +) +def test_web_preflight_refuses_each_unended_calculator_evidence_rule_alone( + world, tmp_path, monkeypatch, fault +): + w, design = complete_web_design(world, tmp_path, monkeypatch) + model = cr.CHEAP[0] + if fault in ("summary", "unresolved"): + path = next((w.runs() / model).glob("run-summary-*.json")) + if fault == "summary": + path.unlink() + else: + summary = json.loads(path.read_text()) + summary["abandoned_running"] = 1 + path.write_text(json.dumps(summary)) + elif fault == "in-flight": + path = w.runs() / model / "in-flight" / "still-running" + path.mkdir(parents=True) + elif fault == "host": + path = w.runs() / "host" / "host.json" + host = json.loads(path.read_text()) + del host["end"] + path.write_text(json.dumps(host)) + elif fault == "chain": + path = w.runs() / model / "run-log.jsonl" + rows = path.read_text().splitlines() + first = json.loads(rows[0]) + first["prev"] = "0" * 64 + rows[0] = json.dumps(first) + path.write_text("\n".join(rows) + "\n") + elif fault == "anchor": + (w.runs() / "anchors.jsonl").write_text("") + elif fault == "push": + path = w.runs() / "anchor-pushes.jsonl" + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[-1]["outcome"] = "error" + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n") + elif fault == "snapshot": + head = cr.line_sha((w.runs() / "anchors.jsonl").read_text().splitlines()[-1]) + (w.runs() / f"anchors.{head}.push-snapshot.jsonl").write_text("altered") + else: + path = w.folder(model, "pe") / f"{w.draw_order[0]}.json" + saved = json.loads(path.read_text()) + saved["totals"]["cost_usd"] += 1 + path.write_text(json.dumps(saved)) + problems = cr.web_preflight(design) + assert len(problems) == 1 and problems[0].startswith( + "the calculator stage has not ended:" + ) + + +def test_web_preflight_counts_all_attempts_stubs_and_late_spend_in_one_fsum( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + # A flat sum retains tiny costs that per-model subtotals would round away. + monkeypatch.setattr(cr, "calculator_evidence", lambda _: [1e16, 1.0, 1.0]) + design["spent_before"] = 1e16 + problems = cr.web_preflight(design) + assert len(problems) == 1 and "recorded spend" in problems[0] + design["spent_before"] = math.fsum([1e16, 1.0, 1.0]) + assert cr.web_preflight(design) == [] + + +def test_web_preflight_applies_the_recorded_spend_tolerance( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + total = design["spent_before"] + design["spent_before"] = total - 1e-6 + assert cr.web_preflight(design) == [] + design["spent_before"] = math.nextafter(total - 1e-6, -math.inf) + assert any("recorded spend" in problem for problem in cr.web_preflight(design)) + + +def test_a_calculator_stage_ended_by_a_terminal_stop_can_precede_web(world): + w = world() + model, sid = cr.CHEAP[0], w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): { + "status": "provider_error", + "error": "BadRequestError: synthetic startup defect", + "turns": 0, + } + } + w.execute() + assert w.events(model, "stop") + assert len([row for row in w.attempts(model) if row["household"] == sid]) == 1 + expected = [ + cr.attempt_cost( + json.loads( + (w.folder(m, row["arm"]) / f"{row['household']}.json").read_text() + ) + ) + for m in w.models + for row in w.attempts(m) + ] + assert math.fsum(cr.calculator_evidence(w.design)) == math.fsum(expected) + + +def test_calculator_end_spend_includes_replaced_stubs_and_provider_attempts(world): + w = world() + model, sid = cr.CHEAP[0], w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = { + (model, first, sid, 1): {"raise_after": True}, + (model, first, sid, 2): { + "status": "provider_error", + "error": "RateLimitError: synthetic", + }, + } + w.execute() + rows = w.attempts(model) + assert { + row["class"] for row in rows if row["household"] == sid and row["arm"] == first + } == {"none", "provider", "ok"} + expected_attempts = ( + N_DRAW + * (len(cr.CHEAP) * len(cr.ARMS) + len(cr.FRONTIER) * len(cr.FRONTIER_ARMS)) + + 2 + ) + assert math.fsum(cr.calculator_evidence(w.design)) == pytest.approx( + expected_attempts * COST + ) + + +def test_calculator_end_spend_includes_late_spend_after_a_closed_deadline( + world, monkeypatch +): + monkeypatch.setattr(cr, "ATTEMPT_DEADLINE", 0.2) + w = world() + model, sid = cr.CHEAP[0], w.draw_order[0] + first = w.run.lanes[model].orders[sid][0] + w.provider.specs = {(model, first, sid, 1): {"sleep": 0.5, "cost": 0.73}} + w.execute() + late = math.fsum(event["cost_usd"] for event in w.events(model, "late-cost")) + assert late == 0.73 + assert not w.events(model, "late-unresolved") + costs = cr.calculator_evidence(w.design) + assert 0.73 in costs + assert math.fsum(costs) == pytest.approx( + w.run.ceiling.spent_usd - w.design["spent_before"] + ) + + +def test_web_design_refuses_frontier_arms_and_unregistered_web_labels( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + design["frontier_arms"] = {} + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the design's arms are not the registered ones" + ] + del design["frontier_arms"] + design["frontier_models"] = [] + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the design's arms are not the registered ones" + ] + del design["frontier_models"] + design["arms"]["pe"] = design["arms"].pop("web") + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the design's arms are not the registered ones" + ] + + +def test_web_preflight_requires_the_registered_whole_run_cap( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + for cap in (150, 270.01, "270", True, None, math.inf): + design["overall_cap"] = cap + assert cr.web_preflight(design) == ["the web design's overall_cap must be 270"] + + +def test_web_stage_lanes_caps_provider_rerun_and_operator_resume( + world, +): + w = world(kind="web", reserve=0.06, caps={m: 0.12 for m in cr.WEB_MODELS}) + sid, model = w.draw_order[0], cr.WEB_MODELS[0] + w.provider.specs = { + (model, "web", sid, 1): { + "status": "provider_error", + "error": "RateLimitError: synthetic transport", + }, + (model, "web+pe3", sid, 1): { + "status": "provider_error", + "error": "AuthenticationError: synthetic account", + "turns": 0, + "resume_after": 0.03, + }, + (model, "web+pe3", sid, 2): {"calls": [{"household": {}}]}, + } + w.execute() + assert run_problems(w) == [] + assert w.models == cr.WEB_MODELS + for name in w.models: + lanes = w.run.lanes[name] + assert lanes.labels == cr.WEB_ARMS + assert [lanes.households(i) for i in (1, 2)] == [ + w.draw_order[::2], + w.draw_order[1::2], + ] + assert any( + row["decision"] == "refuse" for row in w.lines(name, "admission-log.jsonl") + ) + assert lanes.pool.closed + assert any( + row["class"] == "provider" and row["action"] == "rerun" + for row in w.attempts(model) + ) + assert [ + event["event"] + for event in w.events(model) + if event["event"] in ("pause", "resume") + ] == ["pause", "resume"] + assert len(w.run.lanes[model].pool.messages) == 1 + + +def test_web_stage_ceiling_covers_the_whole_run_cap_plus_margin_and_prior_spend(world): + w = world(kind="web", overall_cap=270, spent_before=294.98, reserve=0.01).execute() + assert w.run.ceiling.budget_usd == 295.0 + assert w.run.ceiling.exhausted() + assert all(w.events(model, "ceiling") for model in cr.WEB_MODELS) + assert run_problems(w) == [] + + +def test_web_dry_run_prints_its_models_arms_lanes_caps_and_whole_run_ceiling( + world, tmp_path, monkeypatch, capsys +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = tmp_path / "web-design.json" + path.write_text(json.dumps(design)) + monkeypatch.setattr(pilot, "_require_engine", lambda board: None) + assert cr.main(["run", str(path), "--dry-run"]) == 0 + text = capsys.readouterr().out + assert all( + f"{model}: arms web, web+pe3; lanes 3 and 3" in text for model in cr.WEB_MODELS + ) + assert "hard ceiling $295.00" in text + + +@pytest.mark.parametrize( + "turns,truncated,want", + [(0, False, "none"), (1, False, "provider"), (1, True, "model")], +) +def test_web_request_timeout_keeps_calculator_stage_failure_precedence( + turns, truncated, want +): + # Web requests get no internal timeout retry; the record decides the class. + rec = record( + "provider_error", "Timeout: synthetic", turns=turns, truncated=truncated + ) + assert cr.classify(rec, 1, 2, None)[0] == want + + # ---------------------------------------------------------------- the harness From 38721b4be823d05ebd6eabf42b27bd7452cc05e8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 23:20:04 -0500 Subject: [PATCH 02/31] Preserve calculator model overrides through stage fallback data. Keep the calculator runner's explicit models override compatible with harness models outside the registered roster by assigning those overrides the frontier arms and frontier_arms folder mapping. Store both fallback choices in the calculator stage table and resolve models through its arms_for and folder_key_for helpers. Keep the command line's registered six-model calculator roster unchanged and require web overrides to belong to the registered three-model web roster. Use the same table helpers in the fake record checker and folder fixture. Extend the stage table regression to cover the existing Gemini JSON-contract override and web refusal, while preserving the existing harness conversation test unchanged. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 26 +++++++++++++++++++++++--- tests/test_confirmatory_run.py | 10 ++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index b32451ba..b59bc5eb 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -174,6 +174,24 @@ class Stage: order_stream: str profile_rule: str calculator_label: str + fallback_arms: tuple[str, ...] | None = None + fallback_folder_key: str | None = None + + def arms_for(self, model: str) -> tuple[str, ...]: + """Keep calculator-only explicit model overrides on the frontier arms.""" + if model in self.model_arms: + return self.model_arms[model] + if self.fallback_arms is not None: + return self.fallback_arms + raise KeyError(f"{model}: no model in the {self.kind} stage") + + def folder_key_for(self, model: str) -> str: + """The model's registered mapping, or the calculator override mapping.""" + if model in self.folder_keys: + return self.folder_keys[model] + if self.fallback_folder_key is not None: + return self.fallback_folder_key + raise KeyError(f"{model}: no model in the {self.kind} stage") STAGES = { @@ -185,6 +203,8 @@ class Stage: "order/{model}", "calculator", "pe3", + FRONTIER_ARMS, + "frontier_arms", ), "web": Stage( "web", @@ -1784,8 +1804,8 @@ def __init__( model: Lanes( self, model, - self.stage.model_arms[model], - design[self.stage.folder_keys[model]], + self.stage.arms_for(model), + design[self.stage.folder_key_for(model)], ) for model in self.models } @@ -2757,7 +2777,7 @@ def household_paths(stage): for model in models: if model not in design["caps"] or model not in design["reserves"]: problems.append(f"{model}: no cap or reserve") - for label in stage.model_arms[model]: + for label in stage.arms_for(model): try: arm_profile(model, label) except (ValueError, SystemExit) as error: diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index e8e0dfb1..6cf41135 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -139,11 +139,17 @@ def test_stage_table_registers_the_calculator_and_web_models_arms_and_folders(): } assert calculator.order_stream == "order/{model}" assert calculator.profile_rule == "calculator" + assert calculator.arms_for("gemini-3.1-pro-preview") == cr.FRONTIER_ARMS + assert calculator.folder_key_for("gemini-3.1-pro-preview") == "frontier_arms" assert web.models == ("gpt-6-luna", "gemini-3.5-flash-lite", "claude-sonnet-5.5") assert web.model_arms == {m: ("web", "web+pe3") for m in web.models} assert web.folder_keys == {m: "arms" for m in web.models} assert web.order_stream == "order/web/{model}" assert web.profile_rule == "web" + with pytest.raises(KeyError, match="no model in the web stage"): + web.arms_for("gemini-3.1-pro-preview") + with pytest.raises(KeyError, match="no model in the web stage"): + web.folder_key_for("gemini-3.1-pro-preview") @pytest.mark.parametrize("kind", [None, "", "unknown", [], 7]) @@ -951,7 +957,7 @@ def events(self, model, kind=None) -> list[dict]: ] def folder(self, model, label) -> Path: - arms = self.design[cr.STAGES[self.kind].folder_keys[model]] + arms = self.design[cr.STAGES[self.kind].folder_key_for(model)] return self.runs() / model / arms[label] / "confirm" / "households" @@ -1068,7 +1074,7 @@ def run_problems(world: World) -> list[str]: all_attempts = [] ceiling_events = {} for model in world.models: - labels = cr.STAGES[world.kind].model_arms[model] + labels = cr.STAGES[world.kind].arms_for(model) drawn = cr.orders(master, model, world.draw_order, labels, kind=world.kind) lines = world.attempts(model) events = world.events(model) From fca2a3fe0c0b45061d8c839680d2fedf063a1e1f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 00:49:46 -0500 Subject: [PATCH 03/31] Require integer closure links and test every calculator end refusal. Require calculator archive link numbers to be integers in their sequential positions, and require a successful calculator final push to carry integer return code zero. This keeps booleans from satisfying those web-stage closure gates through Python's numeric equality and agrees with the report's existing integer checks. Add 74 isolated negative cases for summary identity, abandoned-request types, UTC timestamps, attempt and final counts, ended household supervisors, finite closed host spans, every chained log, unsettled and unresolved attempts, record objects and histories, totals and finite nonnegative numeric costs, archive sequence and byte links, run-log reconciliation and ended times, late costs, manifest counts and hashes, complete model and log anchor coverage, and final push identity, return code and chronology. Require one preflight refusal with the named guard's diagnostic for each fault, including the previous nine closure cases. Keep surrounding record digests, archive links, chains, anchor prefixes, push references and snapshots consistent when planting a payload fault, so an unrelated hash failure cannot satisfy its test. Preserve the calculator and web record writers, model rosters, treatment pin and run procedure. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 5 +- tests/test_confirmatory_run.py | 462 ++++++++++++++++++++++++++++++++- 2 files changed, 454 insertions(+), 13 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index b59bc5eb..59435b6a 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2447,7 +2447,9 @@ def finite(value): record, digest = attempt_record(path) links = record["attempts"] if any( - not isinstance(link, dict) or link.get("n") != n + not isinstance(link, dict) + or type(link.get("n")) is not int + or link["n"] != n for n, link in enumerate(links, 1) ): raise ValueError(f"{path}: calculator archive links are invalid") @@ -2545,6 +2547,7 @@ def finite(value): not pushes or pushes[-1].get("anchor") != final_head or pushes[-1].get("outcome") != "success" + or type(pushes[-1].get("returncode")) is not int or pushes[-1].get("returncode") != 0 ): raise ValueError("calculator final anchor has no successful final push") diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 6cf41135..14c11a67 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2485,11 +2485,20 @@ def test_resume_writes_the_operator_s_note(tmp_path): assert note["note"] == "key rotated" -def complete_web_design(world, tmp_path, monkeypatch): +def complete_web_design(world, tmp_path, monkeypatch, *, archives=False): """A web preflight with a genuinely ended synthetic calculator stage.""" from types import SimpleNamespace w = world() + if archives: + model, sid = cr.CHEAP[0], w.draw_order[0] + w.provider.specs = { + (model, "pe", sid, n): { + "status": "provider_error", + "error": "RateLimitError: synthetic", + } + for n in (1, 2) + } w.design.update(complete_preflight_design(w, tmp_path)) w.execute() calculator = tmp_path / "calculator-design.json" @@ -2528,6 +2537,93 @@ def git_pins(args, *a, **kw): return w, design +def rewritten_chain(path, rows, seed): + """Write synthetic evidence while preserving every intended chain link.""" + texts, heads = [], [seed] + for row in rows: + text = json.dumps({**row, "prev": heads[-1]}) + texts.append(text) + heads.append(cr.line_sha(text)) + path.write_text("\n".join(texts) + "\n") + return texts, heads + + +def rewrite_calculator_anchors(w, anchors): + """Keep pushes and snapshots bound when a test changes an anchor's data.""" + path = w.runs() / "anchors.jsonl" + old_heads = [cr.line_sha(line) for line in path.read_text().splitlines()] + texts, heads = rewritten_chain( + path, anchors, cr.chain_seed(w.design["frozen_sha256"], "", path.name) + ) + positions = {head: i for i, head in enumerate(old_heads)} + pushes = w.runs() / "anchor-pushes.jsonl" + rows = [json.loads(line) for line in pushes.read_text().splitlines()] + for row in rows: + position = positions[row["anchor"]] + row["anchor"] = heads[position + 1] + snapshot = w.runs() / f"anchors.{row['anchor']}.push-snapshot.jsonl" + snapshot.write_text("\n".join(texts[: position + 1]) + "\n") + pushes.write_text("\n".join(json.dumps(row) for row in rows) + "\n") + + +def rewrite_calculator_log(w, model, name, rows): + """Update a changed log's anchor prefixes and final pushed snapshot too.""" + path = w.runs() / model / name + _, heads = rewritten_chain( + path, rows, cr.chain_seed(w.design["frozen_sha256"], model, name) + ) + anchors = [ + json.loads(line) + for line in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + for i, anchor in enumerate(anchors): + count = ( + len(rows) + if i == len(anchors) - 1 + else min(anchor["models"][model][name]["lines"], len(rows)) + ) + anchor["models"][model][name] = {"lines": count, "sha256": heads[count]} + rewrite_calculator_anchors(w, anchors) + + +def rewrite_calculator_record(w, model, label, sid, value, *, archive=None): + """Keep a deliberately malformed payload's surrounding hashes valid.""" + folder = w.folder(model, label) + path = folder / ( + f"{sid}.json" if archive is None else f"attempts/{sid}-{archive}.json" + ) + if isinstance(value, bytes): + path.write_bytes(value) + else: + path.write_text(json.dumps(value)) + rows = w.lines(model) + number = archive or max( + row["attempt"] + for row in rows + if row.get("household") == sid and row.get("arm") == label + ) + for row in rows: + if (row.get("household"), row.get("arm"), row.get("attempt")) == ( + sid, + label, + number, + ): + row["sha256"] = cr.sha256_file(path) + if archive is not None: + final_path = folder / f"{sid}.json" + final = json.loads(final_path.read_text()) + final["attempts"][archive - 1]["sha256"] = cr.sha256_file(path) + final_path.write_text(json.dumps(final)) + for row in rows: + if (row.get("household"), row.get("arm"), row.get("attempt")) == ( + sid, + label, + len(final["attempts"]) + 1, + ): + row["sha256"] = cr.sha256_file(final_path) + rewrite_calculator_log(w, model, "run-log.jsonl", rows) + + @pytest.mark.parametrize("fault", ["missing", "wrong", "unreadable", "kind"]) def test_web_preflight_refuses_each_calculator_design_link_fault_alone( world, tmp_path, monkeypatch, fault @@ -2587,21 +2683,21 @@ def test_web_preflight_refuses_each_overlapping_runs_root_alone( @pytest.mark.parametrize( - "fault", + "fault,detail", [ - "summary", - "unresolved", - "in-flight", - "host", - "chain", - "anchor", - "push", - "snapshot", - "record", + ("summary", "calculator stage has no run summary"), + ("unresolved", "calculator summary leaves a request unresolved"), + ("in-flight", "calculator attempts remain in flight"), + ("host", "'end'"), + ("chain", "invalid hash chain"), + ("anchor", "calculator stage has no final anchor"), + ("push", "calculator final anchor has no successful final push"), + ("snapshot", "calculator final push snapshot differs from its anchors"), + ("record", "calculator run log differs from attempt records"), ], ) def test_web_preflight_refuses_each_unended_calculator_evidence_rule_alone( - world, tmp_path, monkeypatch, fault + world, tmp_path, monkeypatch, fault, detail ): w, design = complete_web_design(world, tmp_path, monkeypatch) model = cr.CHEAP[0] @@ -2647,6 +2743,348 @@ def test_web_preflight_refuses_each_unended_calculator_evidence_rule_alone( assert len(problems) == 1 and problems[0].startswith( "the calculator stage has not ended:" ) + assert detail in problems[0], f"calculator closed-evidence rule: {fault}" + + +@pytest.mark.parametrize( + "fault,detail", + [ + ("summary/object", "expected a JSON object"), + ("summary/model", "calculator summary leaves a request unresolved"), + ("summary/boolean-zero", "calculator summary leaves a request unresolved"), + ("summary/missing-abandoned", "calculator summary leaves a request unresolved"), + ("summary/missing-utc", "'finished_at_utc'"), + ("summary/invalid-utc", "does not match format"), + ("summary/null-utc", "strptime() argument 1 must be str"), + ("summary/attempt-count", "calculator summary or log leaves out attempts"), + ("summary/final-count", "calculator summary or log leaves out attempts"), + ("summary/supervisors", "calculator admitted households have not finished"), + ("host/object", "expected a JSON object"), + ("host/reversed-span", "calculator host log has no closed span"), + ("host/boolean-time", "calculator end evidence holds a nonfinite number"), + ("host/infinite-time", "calculator end evidence holds a nonfinite number"), + ("chain/admission", "invalid hash chain"), + ("chain/host", "invalid hash chain"), + ("chain/anchors", "invalid hash chain"), + ("chain/run-row-object", "invalid hash chain"), + ("run/unsettled", "calculator requests are unresolved or unsettled"), + ("run/late-unresolved", "calculator requests are unresolved or unsettled"), + ("payload/object", "invalid calculator attempt record"), + ("payload/prior-attempts", "invalid calculator attempt record"), + ("payload/attempts-type", "calculator attempts must be a list"), + ("payload/missing-attempts", "'attempts'"), + ("payload/totals-type", "calculator totals must be an object"), + ("payload/missing-cost", "calculator cost must be a present number"), + ("payload/boolean-cost", "calculator cost must be a present number"), + ("payload/string-cost", "calculator cost must be a present number"), + ("payload/negative-cost", "calculator cost must be finite and nonnegative"), + ("payload/nan-cost", "calculator cost must be finite and nonnegative"), + ("payload/infinite-cost", "calculator cost must be finite and nonnegative"), + ("payload/overflow-cost", "int too large to convert to float"), + ("payload/malformed-json", "Expecting property name"), + ("payload/invalid-utf8", "utf-8"), + ("archive-payload/object", "invalid calculator attempt record"), + ("archive-payload/boolean-cost", "calculator cost must be a present number"), + ("link/object", "calculator archive links are invalid"), + ("link/number-gap", "calculator archive links are invalid"), + ("link/boolean-number", "calculator archive links are invalid"), + ("link/missing-file", "calculator archives differ from final links"), + ("link/extra-file", "calculator archives differ from final links"), + ("link/path", "calculator archive hash or path differs"), + ("link/hash", "calculator archive hash or path differs"), + ("link/orphan", "calculator archive has no final"), + ("log/duplicate-attempt", "calculator run log differs from attempt records"), + ("log/unknown-attempt", "calculator run log differs from attempt records"), + ("log/omitted-attempt", "calculator summary or log leaves out attempts"), + ("log/reversed-attempt", "calculator attempt has not ended"), + ("log/boolean-time", "calculator end evidence holds a nonfinite number"), + ("log/nonfinite-time", "calculator end evidence holds a nonfinite number"), + ("late/negative-cost", "calculator late spend is negative"), + ("late/boolean-cost", "calculator end evidence holds a nonfinite number"), + ("late/nonfinite-cost", "calculator end evidence holds a nonfinite number"), + ("manifest/missing", "calculator stage needs exactly one manifest"), + ("manifest/extra", "calculator stage needs exactly one manifest"), + ("manifest/hash", "calculator anchor manifest differs"), + ("anchor/missing-model", "calculator anchor does not cover every model"), + ("anchor/extra-model", "calculator anchor does not cover every model"), + ("anchor/manifest", "calculator anchor manifest differs"), + ("anchor/boolean-count", "calculator anchor differs from its logs"), + ("anchor/outside-count", "calculator anchor differs from its logs"), + ("anchor/head", "calculator anchor differs from its logs"), + ("anchor/stale-run", "calculator final anchor omits log lines"), + ("anchor/stale-admission", "calculator final anchor omits log lines"), + ("anchor/stale-host", "calculator final anchor omits log lines"), + ("push/missing", "calculator final anchor has no successful final push"), + ("push/row-object", "invalid hash chain"), + ("push/returncode", "calculator final anchor has no successful final push"), + ( + "push/boolean-returncode", + "calculator final anchor has no successful final push", + ), + ("push/anchor", "calculator final anchor has no successful final push"), + ("push/reversed-time", "calculator final push has invalid times"), + ("push/before-anchor", "calculator final push has invalid times"), + ("push/boolean-time", "calculator end evidence holds a nonfinite number"), + ("push/nonfinite-time", "calculator end evidence holds a nonfinite number"), + ], +) +def test_web_preflight_refuses_each_closed_calculator_evidence_fault_alone( + world, tmp_path, monkeypatch, fault, detail +): + """Every closure refusal identifies its own fault in otherwise valid evidence. + + Log and anchor mutations retain valid chains and matching push snapshots. + Malformed payloads retain their surrounding record and archive digests, so + a generic downstream hash failure cannot stand in for the named refusal. + """ + group, change = fault.split("/", 1) + w, design = complete_web_design( + world, tmp_path, monkeypatch, archives=group in ("archive-payload", "link") + ) + model, sid = cr.CHEAP[0], w.draw_order[0] + folder = w.runs() / model + record_path = w.folder(model, "pe") / f"{sid}.json" + if group == "summary": + path = next(folder.glob("run-summary-*.json")) + value = json.loads(path.read_text()) + if change == "object": + value = [] + elif change == "model": + value["model"] = "different-model" + elif change == "boolean-zero": + value["abandoned_running"] = False + elif change == "missing-abandoned": + del value["abandoned_running"] + elif change == "missing-utc": + del value["finished_at_utc"] + elif change in ("invalid-utc", "null-utc"): + value["finished_at_utc"] = ( + "invalid UTC" if change == "invalid-utc" else None + ) + elif change in ("attempt-count", "final-count"): + value["attempts" if change == "attempt-count" else "final_records"] += 1 + else: + value["households_finished"].remove(sid) + path.write_text(json.dumps(value)) + elif group == "host": + path = w.runs() / "host" / "host.json" + value = json.loads(path.read_text()) + if change == "object": + value = [] + else: + value["end"] = { + "reversed-span": value["start"] - 1, + "boolean-time": True, + "infinite-time": math.inf, + }[change] + path.write_text(json.dumps(value)) + elif group == "chain": + path = { + "admission": folder / "admission-log.jsonl", + "host": w.runs() / "host" / "host-log.jsonl", + "anchors": w.runs() / "anchors.jsonl", + "run-row-object": folder / "run-log.jsonl", + }[change] + texts = path.read_text().splitlines() + if change == "run-row-object": + texts[0] = "[]" + else: + first = json.loads(texts[0]) + first["prev"] = "0" * 64 + texts[0] = json.dumps(first) + path.write_text("\n".join(texts) + "\n") + elif group in ("run", "log", "late"): + rows = w.lines(model) + position = next(i for i, row in enumerate(rows) if "event" not in row) + row = rows[position] + if group == "run": + if change == "unsettled": + row["class"] = "unsettled" + else: + rows.append( + { + "event": "late-unresolved", + "time": row["end"], + "household": row["household"], + "arm": row["arm"], + "attempt": row["attempt"], + "counted_usd": 0.0, + "outcome": "unresolved-at-ledger-close", + } + ) + elif group == "late": + cost = { + "negative-cost": -0.01, + "boolean-cost": True, + "nonfinite-cost": math.inf, + }[change] + rows.append( + { + "event": "late-cost", + "time": row["end"], + "household": row["household"], + "arm": row["arm"], + "attempt": row["attempt"], + "cost_usd": cost, + } + ) + elif change == "duplicate-attempt": + rows.append(dict(row)) + elif change == "unknown-attempt": + row["attempt"] = 99 + elif change == "omitted-attempt": + del rows[position] + elif change == "reversed-attempt": + row["end"] = row["start"] - 1 + else: + row["end"] = True if change == "boolean-time" else math.inf + if group == "log" and change in ("duplicate-attempt", "omitted-attempt"): + summary_path = next(folder.glob("run-summary-*.json")) + summary = json.loads(summary_path.read_text()) + summary["attempts"] = sum("event" not in item for item in rows) + summary_path.write_text(json.dumps(summary)) + rewrite_calculator_log(w, model, "run-log.jsonl", rows) + elif group in ("payload", "archive-payload"): + archive = 1 if group == "archive-payload" else None + path = ( + record_path + if archive is None + else record_path.parent / "attempts" / f"{sid}-1.json" + ) + value = json.loads(path.read_text()) + if change == "object": + value = [] + elif change == "prior-attempts": + value["prior_attempts"] = [] + elif change == "attempts-type": + value["attempts"] = 1 + elif change == "missing-attempts": + del value["attempts"] + elif change == "totals-type": + value["totals"] = [] + elif change == "missing-cost": + del value["totals"]["cost_usd"] + elif change == "malformed-json": + value = b"{" + elif change == "invalid-utf8": + value = b"\xff" + else: + value["totals"]["cost_usd"] = { + "boolean-cost": True, + "string-cost": "0.01", + "negative-cost": -0.01, + "nan-cost": math.nan, + "infinite-cost": math.inf, + "overflow-cost": 10**400, + }[change] + rewrite_calculator_record(w, model, "pe", sid, value, archive=archive) + elif group == "link": + final = json.loads(record_path.read_text()) + archive_path = record_path.parent / "attempts" / f"{sid}-1.json" + if change == "object": + final["attempts"][0] = None + elif change == "number-gap": + final["attempts"][0]["n"] = 2 + elif change == "boolean-number": + final["attempts"][0]["n"] = True + elif change == "missing-file": + archive_path.unlink() + elif change == "extra-file": + (archive_path.parent / f"{sid}-3.json").write_bytes( + archive_path.read_bytes() + ) + elif change == "orphan": + (archive_path.parent / "orphan-1.json").write_bytes( + archive_path.read_bytes() + ) + else: + final["attempts"][0]["file" if change == "path" else "sha256"] = "different" + if change in ("object", "number-gap", "boolean-number", "path", "hash"): + rewrite_calculator_record(w, model, "pe", sid, final) + elif group == "manifest": + path = next(folder.glob("manifest-*.json")) + if change == "missing": + path.unlink() + elif change == "extra": + path.with_name("manifest-extra.json").write_bytes(path.read_bytes()) + else: + value = json.loads(path.read_text()) + value["synthetic_change"] = True + path.write_text(json.dumps(value)) + elif group == "anchor": + anchors = [ + json.loads(line) + for line in (w.runs() / "anchors.jsonl").read_text().splitlines() + ] + final = anchors[-1] + if change == "missing-model": + del final["models"][model] + elif change == "extra-model": + final["models"]["different-model"] = dict(final["models"][model]) + elif change == "manifest": + final["models"][model]["manifest"] = "0" * 64 + elif change in ("boolean-count", "outside-count", "head"): + entry = final["models"][model]["run-log.jsonl"] + if change == "head": + entry["sha256"] = "0" * 64 + else: + entry["lines"] = ( + True if change == "boolean-count" else entry["lines"] + 1 + ) + else: + name = { + "stale-run": "run-log.jsonl", + "stale-admission": "admission-log.jsonl", + "stale-host": "host-log.jsonl", + }[change] + path = w.runs() / ("host" if change == "stale-host" else model) / name + seed = ( + cr.HOST_SEED + if change == "stale-host" + else cr.chain_seed(w.design["frozen_sha256"], model, name) + ) + texts = path.read_text().splitlines() + entry = { + "lines": len(texts) - 1, + "sha256": cr.line_sha(texts[-2]) if len(texts) > 1 else seed, + } + if change == "stale-host": + final[name] = entry + else: + final["models"][model][name] = entry + rewrite_calculator_anchors(w, anchors) + else: + path = w.runs() / "anchor-pushes.jsonl" + pushes = [json.loads(line) for line in path.read_text().splitlines()] + if change == "missing": + pushes = [] + elif change == "row-object": + pushes[-1] = [] + else: + final = pushes[-1] + if change == "returncode": + final["returncode"] = 1 + elif change == "boolean-returncode": + final["returncode"] = False + elif change == "anchor": + final["anchor"] = "0" * 64 + elif change == "reversed-time": + final["ended"] = final["started"] - 1 + elif change == "before-anchor": + final["started"] = final["anchor_time"] - 1 + else: + final["ended"] = True if change == "boolean-time" else math.inf + path.write_text( + "\n".join(json.dumps(row) for row in pushes) + ("\n" if pushes else "") + ) + problems = cr.web_preflight(design) + assert len(problems) == 1, ( + f"one calculator closed-evidence refusal for {fault}: {problems}" + ) + assert problems[0].startswith("the calculator stage has not ended:") + assert detail in problems[0], ( + f"calculator closed-evidence rule {fault}: {problems[0]}" + ) def test_web_preflight_counts_all_attempts_stubs_and_late_spend_in_one_fsum( From eed0ba55da7a068729b2006dda7f1adf0cbe2f9c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 07:48:43 -0500 Subject: [PATCH 04/31] Require the checked LiteLLM version before Gemini web startup. Refuse incompatible Gemini web designs during preflight so the search patch cannot terminate a run after startup writes. Verify the runner applies its patch before the first Gemini completion and that both web arms retain search and every function declaration in serialized HTTP requests. Restore LiteLLM class descriptors after each runner test. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 17 +++- tests/test_confirmatory_run.py | 168 +++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 59435b6a..0520eea6 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -101,7 +101,11 @@ treatment commit's (c0a2885b), its tree is clean at the design's run commit, the machine is on AC power with the lid open, and the runs folder holds no records: a run is never resumed. ``--workers`` must equal the design's -engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the +engine workers per model. A Gemini web model also requires the exact LiteLLM +version checked by the treatment's search patch at preflight. During execute, +after setup-start and before the completion factory or any completion call, +the runner applies that patch so search survives beside function tools. +An interrupt (SIGINT or SIGTERM) only notes the signal; the main loop then stops every model's lanes with cause ``interrupt`` and lets the conversations under way end. A second interrupt exits at once until cleanup ignores signals before its final check. Signals @@ -113,6 +117,7 @@ import argparse import hashlib +import importlib.metadata import json import math import os @@ -2663,6 +2668,16 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: problems.extend(stage_problems(design)) if stage.kind == "web": problems.extend(web_preflight(design)) + if any(pilot.model_profile(model).provider == "gemini" for model in models): + try: + version = importlib.metadata.version("litellm") + except importlib.metadata.PackageNotFoundError: + version = "not installed" + if version != pilot.GEMINI_SEARCH_PATCH_LITELLM: + problems.append( + f"litellm {version} is installed; the Gemini search patch was " + f"checked against {pilot.GEMINI_SEARCH_PATCH_LITELLM} only" + ) try: no_tools_root = Path(design["no_tools"]["runs"]).resolve() calculator_root = Path(design["runs"]).resolve() diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 14c11a67..98ab7e24 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -58,6 +58,21 @@ TRIGGER_BOUND = 5.0 +@pytest.fixture(autouse=True) +def keep_litellm_gemini_config(monkeypatch): + """Restore the class descriptors after every test, including fake web runs.""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + for name in ( + "_resolve_search_tool_conflict", + "_drop_search_tools_mixed_with_functions", + ): + monkeypatch.setattr(VertexGeminiConfig, name, VertexGeminiConfig.__dict__[name]) + return VertexGeminiConfig + + # ---------------------------------------------------------------- streams @@ -176,6 +191,84 @@ def test_web_arm_profile_keeps_search_the_benchmark_rule_and_its_calculator( ) +def first_gemini_http_body(monkeypatch, label): + """Run the harness's initial request through the real LiteLLM HTTP path.""" + import httpx + + class BodyCaptured(BaseException): + pass + + captured = [] + + def capture(self, request, **kwargs): + assert request.method == "POST" + assert request.url.host == "generativelanguage.googleapis.com" + captured.append(json.loads(request.content)) + raise BodyCaptured + + monkeypatch.setenv("GEMINI_API_KEY", "synthetic-no-network") + monkeypatch.setattr(httpx.Client, "send", capture) + profile = cr.arm_profile("gemini-3.5-flash-lite", label) + # run_household calls build_initial_request and assembles the actual first + # turn, including submit_outputs and the arm's calculator/search tools. + with pytest.raises(BodyCaptured): + pilot.run_household( + scenario("h00"), + VARIABLES, + lambda arguments: pytest.fail("the captured request cannot call a tool"), + cr.default_completion(profile), + pilot.Spend(1.0), + profile=profile, + ) + assert len(captured) == 1 + return captured[0] + + +@pytest.mark.parametrize("label", cr.WEB_ARMS) +def test_gemini_web_http_body_keeps_search_beside_function_declarations( + monkeypatch, label, world +): + world(kind="web", models=("gemini-3.5-flash-lite",)).execute() + body = first_gemini_http_body(monkeypatch, label) + assert any("googleSearch" in tool for tool in body["tools"]), ( + "the patched Gemini HTTP body must offer googleSearch" + ) + functions = { + function["name"] + for tool in body["tools"] + for function in tool.get("function_declarations", []) + } + assert functions == ( + {"submit_outputs", "calculate_household"} + if label == "web+pe3" + else {"submit_outputs"} + ), "the Gemini HTTP body must retain every function declaration" + + +@pytest.mark.parametrize("label", cr.WEB_ARMS) +def test_unpatched_gemini_web_http_body_drops_search_beside_functions( + monkeypatch, label, keep_litellm_gemini_config +): + config = keep_litellm_gemini_config + original = { + name: config.__dict__[name] + for name in ( + "_resolve_search_tool_conflict", + "_drop_search_tools_mixed_with_functions", + ) + } + pilot.allow_gemini_search_with_functions() + for name, descriptor in original.items(): + monkeypatch.setattr(config, name, descriptor) + body = first_gemini_http_body(monkeypatch, label) + assert not any("googleSearch" in tool for tool in body["tools"]), ( + "the unpatched control must expose LiteLLM's removal of googleSearch" + ) + assert any("function_declarations" in tool for tool in body["tools"]), ( + "the unpatched control must still reach Gemini's function declarations" + ) + + @pytest.mark.parametrize( "label,changed,message", [ @@ -2537,6 +2630,49 @@ def git_pins(args, *a, **kw): return w, design +def test_web_gemini_preflight_refuses_the_wrong_litellm_alone_before_writes( + world, tmp_path, monkeypatch, capsys +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + real_version = cr.importlib.metadata.version + monkeypatch.setattr( + cr.importlib.metadata, + "version", + lambda name: "9.9.9" if name == "litellm" else real_version(name), + ) + problem = ( + "litellm 9.9.9 is installed; the Gemini search patch was " + f"checked against {pilot.GEMINI_SEARCH_PATCH_LITELLM} only" + ) + assert cr.preflight(design, cr.WEB_MODELS) == [problem], ( + "an otherwise valid Gemini web design must have only the litellm refusal" + ) + path = tmp_path / "web-design.json" + path.write_text(json.dumps(design)) + monkeypatch.setattr(pilot, "_require_engine", lambda board: None) + assert cr.main(["run", str(path)]) == 1 + assert f"The run cannot start:\n- {problem}" in capsys.readouterr().out + assert not Path(design["runs"]).exists(), ( + "the litellm refusal must precede every web startup write" + ) + + +def test_calculator_gemini_preflight_has_no_web_litellm_version_check( + world, tmp_path, monkeypatch +): + w, _ = complete_web_design(world, tmp_path, monkeypatch) + calculator = {**w.design, "runs": str(tmp_path / "fresh-calculator-runs")} + real_version = cr.importlib.metadata.version + monkeypatch.setattr( + cr.importlib.metadata, + "version", + lambda name: "9.9.9" if name == "litellm" else real_version(name), + ) + assert cr.preflight(calculator, ("gemini-3.5-flash-lite",)) == [], ( + "a calculator design must remain valid regardless of the web patch version" + ) + + def rewritten_chain(path, rows, seed): """Write synthetic evidence while preserving every intended chain link.""" texts, heads = [], [seed] @@ -3211,6 +3347,38 @@ def test_web_preflight_requires_the_registered_whole_run_cap( assert cr.web_preflight(design) == ["the web design's overall_cap must be 270"] +def test_web_execute_patches_gemini_before_its_first_completion( + world, keep_litellm_gemini_config +): + config = keep_litellm_gemini_config + original = { + name: config.__dict__[name] + for name in ( + "_resolve_search_tool_conflict", + "_drop_search_tools_mixed_with_functions", + ) + } + w = world(kind="web", models=("gemini-3.5-flash-lite",)) + observed = [] + + def completion(**kwargs): + assert all(config.__dict__[name] is not value for name, value in original.items()), ( + "execute must install both Gemini patch methods before completion" + ) + observed.append(True) + + def converse(lanes, sid, label, tool_call, spend): + lanes.completion() + return w.provider(lanes, sid, label, tool_call, spend) + + w.run.completion_for = lambda profile: completion + w.run.converse = converse + w.execute() + assert len(observed) == N_DRAW * len(cr.WEB_ARMS), ( + "the patched Gemini completion must run for every web arm and household" + ) + + def test_web_stage_lanes_caps_provider_rerun_and_operator_resume( world, ): From 78315ddaf5569b69dfcaeb031ae80d4ccf2688f0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 08:43:22 -0500 Subject: [PATCH 05/31] Enforce registered stage budgets before any provider request. Require complete prior spend, the fixed calculator cap, starts below the ceiling, and proportional caps with three-times reserves for each stage and linked calculator design. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 89 +++++++++++++- tests/test_confirmatory_run.py | 219 ++++++++++++++++++++++++++++++++- 2 files changed, 298 insertions(+), 10 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 0520eea6..bb698cc9 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -273,6 +273,12 @@ def stage_for(design: dict) -> Stage: HOST_LOG_GAP = 90.0 # seconds SAMPLE_SECONDS = 30.0 CEILING_MARGIN = 25.0 # dollars above the overall cap +CALCULATOR_OVERALL_CAP = 150.0 +CALCULATOR_CAP = 145.0 +WEB_OVERALL_CAP = 270.0 +WEB_CAP = 120.0 +CAP_TOLERANCE = 0.005 # dollars: a cap rounded to the cent +RESERVE_TOLERANCE = 0.0005 # dollars: a reserve rounded to a tenth of a cent ADMISSION_SLACK = 1e-9 # dollars EXCERPT_LEAD = 86_400.0 # host logs are kept from a day before the run's start WAIT_SECONDS = 1.0 # how often a waiting lane looks again @@ -956,7 +962,9 @@ class Ceiling(pilot.Spend): def __init__(self, ceiling_usd: float, spent_before: float): super().__init__(ceiling_usd, spent_before) - self.crossed_at: float | None = None + self.crossed_at: float | None = ( + clock() if spent_before >= ceiling_usd else None + ) self.closed = False # the records are final: spend is no longer counted self.unrecorded_usd = 0.0 # spend after that, in no record @@ -2578,7 +2586,7 @@ def web_preflight(design: dict) -> list[str]: ) problems = [] - if design.get("overall_cap") != 270: + if design.get("overall_cap") != WEB_OVERALL_CAP: problems.append("the web design's overall_cap must be 270") try: link = design["calculator"] @@ -2591,6 +2599,13 @@ def web_preflight(design: dict) -> list[str]: return [*problems, "the linked calculator design's kind must be calculator"] except (OSError, ValueError, KeyError, TypeError, RecursionError) as error: return [*problems, f"the calculator design cannot be read: {error}"] + if calculator.get("overall_cap") != CALCULATOR_OVERALL_CAP: + problems.append("the linked calculator design's overall_cap must be 150") + problems.extend(f"linked calculator: {p}" for p in stage_start_problems(calculator)) + problems.extend( + f"linked calculator: {p}" + for p in allocation_problems(calculator, STAGES["calculator"]) + ) for field in SHARED_STAGE_FIELDS: if field not in calculator or design.get(field) != calculator[field]: problems.append(f"web and calculator designs differ in {field}") @@ -2613,7 +2628,8 @@ def web_preflight(design: dict) -> list[str]: except (OSError, ValueError, KeyError, TypeError, RuntimeError) as error: problems.append(f"the web runs roots are invalid: {error}") try: - costs = calculator_evidence(calculator) + calculator_costs = calculator_evidence(calculator) + costs = list(calculator_costs) spec = design["no_tools"] for model in saved_children(Path(spec["runs"])): if listed_is_dir(model): @@ -2624,7 +2640,14 @@ def web_preflight(design: dict) -> list[str]: / "households" ) costs.extend(saved) - spent, declared = exact_spend(costs), float(design["spent_before"]) + calculator_spent_before = float(calculator["spent_before"]) + if not math.isfinite(calculator_spent_before) or calculator_spent_before < 0: + raise ValueError("calculator spent_before must be finite and nonnegative") + spent = max( + exact_spend([calculator_spent_before, *calculator_costs]), + exact_spend(costs), + ) + declared = float(design["spent_before"]) if not 0 <= declared < math.inf or declared < spent - 1e-6: problems.append( f"web spent_before ${declared} is below no-tools and calculator " @@ -2643,6 +2666,60 @@ def web_preflight(design: dict) -> list[str]: return problems +def stage_start_problems(design: dict) -> list[str]: + """No provider request may start once the whole-run ceiling is reached.""" + try: + spent_before = float(design["spent_before"]) + ceiling = float(design["overall_cap"]) + CEILING_MARGIN + if math.isfinite(spent_before) and spent_before < ceiling: + return [] + except (KeyError, TypeError, ValueError, OverflowError): + pass + return ["spent_before must be finite and below the stage ceiling"] + + +def allocation_problems(design: dict, stage: Stage) -> list[str]: + """Check the registered proportional cap split and three-times reserves. + + The independent report also compares expected costs with its development + projection. The runner has no access to that development evidence. + """ + problems = [] + expected = {} + for model in stage.models: + try: + value = design["expected_costs"][model] + cost = float(value) + if isinstance(value, bool) or not math.isfinite(cost) or cost <= 0: + raise ValueError("expected cost is not finite and positive") + expected[model] = cost + except (KeyError, TypeError, ValueError, OverflowError): + problems.append(f"{model}: expected_costs must be finite and positive") + if problems: + return problems + total = sum(expected.values()) + if not math.isfinite(total): + return ["the expected costs do not sum to a finite positive amount"] + stage_cap = WEB_CAP if stage.kind == "web" else CALCULATOR_CAP + for model, cost in expected.items(): + for field, want, tolerance, rule in ( + ("caps", stage_cap * cost / total, CAP_TOLERANCE, "split"), + ("reserves", 3 * cost / HOUSEHOLDS, RESERVE_TOLERANCE, "rule"), + ): + try: + value = design[field][model] + got = float(value) + if isinstance(value, bool) or not math.isfinite(got): + raise ValueError("allocation is not finite") + except (KeyError, TypeError, ValueError, OverflowError): + problems.append(f"{model}: {field} needs a finite allocation") + continue + if abs(got - want) > tolerance: + name = "cap" if field == "caps" else "reserve" + problems.append(f"{model}: {name} ${got}, and the {rule} gives ${want}") + return problems + + def preflight(design: dict, models: tuple[str, ...]) -> list[str]: """What stops the run from starting.""" missing = [k for k in REQUIRED if k not in design] @@ -2655,6 +2732,10 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if stage.kind == "calculator" and "frontier_arms" not in design: return ["the design has no frontier_arms"] problems = [] + if stage.kind == "calculator" and design.get("overall_cap") != CALCULATOR_OVERALL_CAP: + problems.append("the calculator design's overall_cap must be 150") + problems.extend(stage_start_problems(design)) + problems.extend(allocation_problems(design, stage)) from confirmatory_no_tools import ( BOARD_INPUT_ERRORS, _board_input, diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 98ab7e24..b6967467 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2434,6 +2434,10 @@ def complete_preflight_design(w, tmp_path): baseline.write_text('{"exclusions": []}\n') design = dict( w.design, + overall_cap=150.0, + expected_costs={m: 1.0 for m in cr.STAGES["calculator"].models}, + caps={m: 145.0 / 6 for m in cr.STAGES["calculator"].models}, + reserves={m: 3.0 / 120 for m in cr.STAGES["calculator"].models}, eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, baseline_exclusions={"path": str(baseline), "sha256": cr.sha256_file(baseline)}, treatment_harness=str(cr.ROOT), @@ -2582,7 +2586,11 @@ def complete_web_design(world, tmp_path, monkeypatch, *, archives=False): """A web preflight with a genuinely ended synthetic calculator stage.""" from types import SimpleNamespace - w = world() + w = world( + overall_cap=150.0, + caps={m: 145.0 / 6 for m in cr.STAGES["calculator"].models}, + reserve=3.0 / 120, + ) if archives: model, sid = cr.CHEAP[0], w.draw_order[0] w.provider.specs = { @@ -2602,10 +2610,13 @@ def complete_web_design(world, tmp_path, monkeypatch, *, archives=False): runs=str(tmp_path / "web-runs"), arms={a: a.replace("+", "-") for a in cr.WEB_ARMS}, calculator={"path": str(calculator), "sha256": cr.sha256_file(calculator)}, - caps={m: 40.0 for m in cr.WEB_MODELS}, - reserves={m: 0.06 for m in cr.WEB_MODELS}, + expected_costs=dict(zip(cr.WEB_MODELS, (3.0, 5.0, 12.0))), + caps=dict(zip(cr.WEB_MODELS, (18.0, 30.0, 72.0))), + reserves=dict(zip(cr.WEB_MODELS, (0.075, 0.125, 0.3))), overall_cap=270.0, - spent_before=math.fsum(cr.calculator_evidence(w.design)), + spent_before=math.fsum( + [w.design["spent_before"], *cr.calculator_evidence(w.design)] + ), ) del design["frontier_arms"] monkeypatch.setattr( @@ -2630,6 +2641,196 @@ def git_pins(args, *a, **kw): return w, design +def fresh_stage_design(w, web, tmp_path, kind): + """Keep every unrelated preflight input valid for an isolated budget fault.""" + if kind == "web": + return copy.deepcopy(web), cr.WEB_MODELS + design = copy.deepcopy(w.design) + design["runs"] = str(tmp_path / "fresh-calculator-runs") + return design, cr.STAGES["calculator"].models + + +def test_web_spent_before_covers_calculator_declaration_and_all_saved_costs( + world, tmp_path, monkeypatch +): + w, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["spent_before"] = 6.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + final = next(Path(design["no_tools"]["runs"]).rglob("households/*.json")) + record = json.loads(final.read_text()) + record["totals"]["cost_usd"] = 1.0 + final.write_text(json.dumps(record)) + monkeypatch.setattr(cr, "calculator_evidence", lambda _: [88.0]) + for refused in (89.0, 93.99): + design["spent_before"] = refused + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and "recorded spend $94.0" in problems[0], ( + "web spent_before must cover the calculator declaration plus its costs" + ) + design["spent_before"] = 94.0 + assert cr.preflight(design, cr.WEB_MODELS) == [], ( + "web spent_before accepts the complete $94 declared-and-recorded floor" + ) + calculator["spent_before"] = 0.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + design["spent_before"] = 88.0 + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and "recorded spend $89.0" in problems[0], ( + "web spent_before also covers the complete saved-cost inventory" + ) + + +def test_calculator_preflight_requires_the_registered_150_dollar_cap( + world, tmp_path, monkeypatch +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, "calculator") + assert cr.preflight(design, models) == [] + for cap in (149.99, 150.01, 270.0): + design["overall_cap"] = cap + assert cr.preflight(design, models) == [ + "the calculator design's overall_cap must be 150" + ], "calculator preflight fixes the whole-run cap at $150" + + +def test_linked_calculator_design_requires_the_registered_150_dollar_cap( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["overall_cap"] = 270.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, cr.WEB_MODELS) == [ + "the linked calculator design's overall_cap must be 150" + ], "the web preflight also fixes its linked calculator cap at $150" + + +def test_linked_calculator_design_refuses_a_start_at_its_175_dollar_ceiling( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + design["spent_before"] = 200.0 + for spent, refused in ((174.99, False), (175.0, True), (175.01, True)): + calculator["spent_before"] = spent + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, cr.WEB_MODELS) == ( + ["linked calculator: spent_before must be finite and below the stage ceiling"] + if refused else [] + ), "a linked calculator design must have started below its $175 ceiling" + + +@pytest.mark.parametrize("field", ["expected_costs", "caps", "reserves"]) +def test_linked_calculator_design_requires_registered_allocations( + world, tmp_path, monkeypatch, field +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + model = cr.CHEAP[0] + calculator[field][model] = ( + 0.0 if field == "expected_costs" else calculator[field][model] + 0.1 + ) + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and problems[0].startswith("linked calculator: "), ( + "a linked calculator design must use its registered allocations" + ) + + +@pytest.mark.parametrize("kind,ceiling", [("calculator", 175.0), ("web", 295.0)]) +def test_stage_preflight_refuses_start_at_or_above_the_ceiling( + world, tmp_path, monkeypatch, kind, ceiling +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + design["spent_before"] = ceiling - 0.01 + assert cr.preflight(design, models) == [], "a start below the ceiling is accepted" + for amount in (ceiling, ceiling + 0.01): + design["spent_before"] = amount + assert cr.preflight(design, models) == [ + "spent_before must be finite and below the stage ceiling" + ], "the stage preflight refuses a start at or above its ceiling" + + +@pytest.mark.parametrize("ceiling", [175.0, 295.0]) +def test_ceiling_is_exhausted_at_construction_at_or_above_its_limit(ceiling): + below = cr.Ceiling(ceiling, ceiling - 0.01) + assert not below.exhausted() and below.crossed_at is None + for amount in (ceiling, ceiling + 0.01): + spent = cr.Ceiling(ceiling, amount) + assert spent.exhausted() and spent.crossed_at is not None, ( + "a ceiling reached before construction is already exhausted" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("cost", [None, 0.0, -1.0, math.inf, math.nan, True]) +def test_stage_allocation_requires_each_expected_cost_to_be_finite_and_positive( + world, tmp_path, monkeypatch, kind, cost +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + model = models[0] + if cost is None: + del design["expected_costs"][model] + else: + design["expected_costs"][model] = cost + assert cr.preflight(design, models) == [ + f"{model}: expected_costs must be finite and positive" + ], "each stage model needs a finite positive expected cost" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_requires_caps_to_follow_the_proportional_split( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + model = models[0] + design["caps"][model] += 0.01 + problems = cr.preflight(design, models) + assert len(problems) == 1 and "and the split gives" in problems[0], ( + "each stage cap must follow its registered proportional split" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_requires_three_times_expected_cost_reserves( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + model = models[0] + design["reserves"][model] += 0.001 + problems = cr.preflight(design, models) + assert len(problems) == 1 and "and the rule gives" in problems[0], ( + "each stage reserve is three times its expected cost over 120 households" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_accepts_the_reports_cap_and_reserve_tolerances( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + design["caps"][models[0]] += 0.0049 + design["reserves"][models[0]] += 0.00049 + assert cr.preflight(design, models) == [], ( + "stage allocation checks use the report's $0.005/$0.0005 tolerances" + ) + + def test_web_gemini_preflight_refuses_the_wrong_litellm_alone_before_writes( world, tmp_path, monkeypatch, capsys ): @@ -2804,7 +3005,7 @@ def test_web_preflight_refuses_each_shared_input_difference_alone( def test_web_preflight_refuses_each_overlapping_runs_root_alone( world, tmp_path, monkeypatch, root, direction ): - w, design = complete_web_design(world, tmp_path, monkeypatch) + _, design = complete_web_design(world, tmp_path, monkeypatch) other = Path(design["no_tools"]["runs"] if root == "no_tools" else w.design["runs"]) design["runs"] = str( other @@ -3226,7 +3427,13 @@ def test_web_preflight_refuses_each_closed_calculator_evidence_fault_alone( def test_web_preflight_counts_all_attempts_stubs_and_late_spend_in_one_fsum( world, tmp_path, monkeypatch ): - _, design = complete_web_design(world, tmp_path, monkeypatch) + w, design = complete_web_design(world, tmp_path, monkeypatch) + calculator = json.loads(Path(design["calculator"]["path"]).read_text()) + calculator["spent_before"] = 0.0 + Path(design["calculator"]["path"]).write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file( + Path(design["calculator"]["path"]) + ) # A flat sum retains tiny costs that per-model subtotals would round away. monkeypatch.setattr(cr, "calculator_evidence", lambda _: [1e16, 1.0, 1.0]) design["spent_before"] = 1e16 From 4dc830b3d1d8ff0a9f7b011ae3926458f11a07b0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 08:44:20 -0500 Subject: [PATCH 06/31] Keep the expected-cost refusal isolated from allocation arithmetic. Set caps and reserves from finite nonpositive synthetic costs so that the positivity gate is the sole failing rule. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index b6967467..53741f90 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2785,6 +2785,17 @@ def test_stage_allocation_requires_each_expected_cost_to_be_finite_and_positive( del design["expected_costs"][model] else: design["expected_costs"][model] = cost + if isinstance(cost, float) and math.isfinite(cost): + total = sum(design["expected_costs"].values()) + stage_cap = 120.0 if kind == "web" else 145.0 + design["caps"] = { + m: stage_cap * value / total + for m, value in design["expected_costs"].items() + } + design["reserves"] = { + m: 3 * value / 120 + for m, value in design["expected_costs"].items() + } assert cr.preflight(design, models) == [ f"{model}: expected_costs must be finite and positive" ], "each stage model needs a finite positive expected cost" From 91065452f6d22d98b3f860d48f43af5b2b416cc9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 08:45:48 -0500 Subject: [PATCH 07/31] Match budget fixtures to their synthetic household count. Derive reserve fixtures from the patched household count and retain the calculator world needed by root-overlap tests. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 53741f90..93c7da47 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2437,7 +2437,7 @@ def complete_preflight_design(w, tmp_path): overall_cap=150.0, expected_costs={m: 1.0 for m in cr.STAGES["calculator"].models}, caps={m: 145.0 / 6 for m in cr.STAGES["calculator"].models}, - reserves={m: 3.0 / 120 for m in cr.STAGES["calculator"].models}, + reserves={m: 3.0 / cr.HOUSEHOLDS for m in cr.STAGES["calculator"].models}, eligible_pool={"path": str(pool), "sha256": cr.sha256_file(pool)}, baseline_exclusions={"path": str(baseline), "sha256": cr.sha256_file(baseline)}, treatment_harness=str(cr.ROOT), @@ -2589,7 +2589,7 @@ def complete_web_design(world, tmp_path, monkeypatch, *, archives=False): w = world( overall_cap=150.0, caps={m: 145.0 / 6 for m in cr.STAGES["calculator"].models}, - reserve=3.0 / 120, + reserve=3.0 / cr.HOUSEHOLDS, ) if archives: model, sid = cr.CHEAP[0], w.draw_order[0] @@ -2612,7 +2612,10 @@ def complete_web_design(world, tmp_path, monkeypatch, *, archives=False): calculator={"path": str(calculator), "sha256": cr.sha256_file(calculator)}, expected_costs=dict(zip(cr.WEB_MODELS, (3.0, 5.0, 12.0))), caps=dict(zip(cr.WEB_MODELS, (18.0, 30.0, 72.0))), - reserves=dict(zip(cr.WEB_MODELS, (0.075, 0.125, 0.3))), + reserves={ + m: 3.0 * cost / cr.HOUSEHOLDS + for m, cost in zip(cr.WEB_MODELS, (3.0, 5.0, 12.0)) + }, overall_cap=270.0, spent_before=math.fsum( [w.design["spent_before"], *cr.calculator_evidence(w.design)] @@ -2793,7 +2796,7 @@ def test_stage_allocation_requires_each_expected_cost_to_be_finite_and_positive( for m, value in design["expected_costs"].items() } design["reserves"] = { - m: 3 * value / 120 + m: 3 * value / cr.HOUSEHOLDS for m, value in design["expected_costs"].items() } assert cr.preflight(design, models) == [ @@ -3016,7 +3019,7 @@ def test_web_preflight_refuses_each_shared_input_difference_alone( def test_web_preflight_refuses_each_overlapping_runs_root_alone( world, tmp_path, monkeypatch, root, direction ): - _, design = complete_web_design(world, tmp_path, monkeypatch) + w, design = complete_web_design(world, tmp_path, monkeypatch) other = Path(design["no_tools"]["runs"] if root == "no_tools" else w.design["runs"]) design["runs"] = str( other From f92c90ae09a0a27fe7a57b8a89646cd5763a15e1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 08:51:11 -0500 Subject: [PATCH 08/31] Sum allocation projections with the report's rounding rule. Use one math.fsum for expected costs and refuse overflow before cap or reserve calculations for both stages. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 5 ++++- tests/test_confirmatory_run.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index bb698cc9..d693b798 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2697,7 +2697,10 @@ def allocation_problems(design: dict, stage: Stage) -> list[str]: problems.append(f"{model}: expected_costs must be finite and positive") if problems: return problems - total = sum(expected.values()) + try: + total = math.fsum(expected.values()) + except OverflowError: + total = math.inf if not math.isfinite(total): return ["the expected costs do not sum to a finite positive amount"] stage_cap = WEB_CAP if stage.kind == "web" else CALCULATOR_CAP diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 93c7da47..8186f0a5 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2845,6 +2845,20 @@ def test_stage_allocation_accepts_the_reports_cap_and_reserve_tolerances( ) +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_stage_allocation_refuses_an_overflowing_expected_cost_total(kind): + stage = cr.STAGES[kind] + design = { + "kind": kind, + "expected_costs": {model: 1e308 for model in stage.models}, + "caps": {model: 20.0 for model in stage.models}, + "reserves": {model: 0.1 for model in stage.models}, + } + assert cr.allocation_problems(design, stage) == [ + "the expected costs do not sum to a finite positive amount" + ], "each stage allocation must refuse an overflowing expected-cost total" + + def test_web_gemini_preflight_refuses_the_wrong_litellm_alone_before_writes( world, tmp_path, monkeypatch, capsys ): From 874c25f35c1cd7f0639198fdb3c4a3def0609915 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 16:48:28 -0500 Subject: [PATCH 09/31] Require explicit valid costs when reading saved attempts. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 2 +- scripts/confirmatory_run.py | 28 ++++++++++++--- tests/test_confirmatory_run.py | 59 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 90c644c8..67a10b36 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -989,7 +989,7 @@ def no_tool(arguments): cls, _ = classify(record, 0.0, 0.0, None) outcome = action(cls, n) self.say( - f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record):.6f}" + f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record, saved=False):.6f}" ) if self.interrupted.is_set(): self.states[model] = "interrupted" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index d693b798..400abb1d 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -573,8 +573,28 @@ def action(cls: str, attempt: int) -> str: return "stands" -def attempt_cost(record: dict) -> float: - return float((record.get("totals") or {}).get("cost_usd") or 0.0) +def attempt_cost(record: dict, *, saved: bool = True) -> float: + """A saved attempt must account explicitly for its whole recorded cost. + + Live accounting keeps its existing behavior: the harness writes the total, + including error/deadline stubs, and malformed output still gets recorded. + Validation of evidence read back from disk must never invent a zero cost. + """ + if not saved: + return float((record.get("totals") or {}).get("cost_usd") or 0.0) + totals = record.get("totals") + if not isinstance(totals, dict): + raise ValueError("attempt totals must be an object with cost_usd") + value = totals.get("cost_usd") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("attempt cost_usd must be a present number") + try: + cost = float(value) + except OverflowError as error: + raise ValueError("attempt cost_usd must be finite and nonnegative") from error + if not math.isfinite(cost) or cost < 0: + raise ValueError("attempt cost_usd must be finite and nonnegative") + return cost _clock_lock = threading.Lock() @@ -1305,7 +1325,7 @@ def end_attempt( abandoned at its deadline.""" with self.cond: end = clock() - self.spent[sid] += attempt_cost(record) + self.spent[sid] += attempt_cost(record, saved=False) attempt = Attempt(label, n, start, end, record) if retained is not None: retained.append(attempt) # recovery owns it before any event I/O @@ -1546,7 +1566,7 @@ def passes( write_json(in_flight / f"{label}-{n}.json", record) self.run.say( f"{self.model} lane {lane} {sid} {label} #{n}: " - f"{record.get('status')} ${attempt_cost(record):.4f} " + f"{record.get('status')} ${attempt_cost(record, saved=False):.4f} " f"(total ${self.run.ceiling.spent_usd:.2f})" ) if not done: diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 8186f0a5..bbc103b6 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2401,6 +2401,45 @@ def test_the_preflight_names_a_missing_field(): assert cr.preflight({}, cr.CHEAP) == [f"the design has no {', '.join(cr.REQUIRED)}"] +INVALID_SAVED_COSTS = [ + "missing-totals", "missing-cost", None, False, True, "0", math.nan, -0.01, +] + + +def invalidate_saved_cost(record, fault): + if fault == "missing-totals": + record.pop("totals") + elif fault == "missing-cost": + record["totals"].pop("cost_usd") + else: + record["totals"]["cost_usd"] = fault + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("record_kind", ["final", "replaced", "stub"]) +@pytest.mark.parametrize("fault", INVALID_SAVED_COSTS) +def test_saved_attempt_cost_requires_an_explicit_valid_total(kind, record_kind, fault): + record = { + "scenario_id": "last-household", "kind": kind, + "status": "harness_error" if record_kind == "stub" else "complete", + "attempts": [], "totals": {"cost_usd": 0.0}, + } + assert cr.attempt_cost(record) == 0.0 + invalidate_saved_cost(record, fault) + refused = False + try: + cr.attempt_cost(record) + except ValueError: + refused = True + assert refused, "saved attempt costs require a present nonboolean finite nonnegative number" + + +def test_live_attempt_cost_keeps_recording_a_missing_total(): + assert cr.attempt_cost({}, saved=False) == 0.0, ( + "live accounting retains records before saved-evidence validation" + ) + + def complete_preflight_design(w, tmp_path): """Each calculator preflight starts after standing no-tools finals that pass the no-tools script's treatment checks: built through the harness from the @@ -2463,6 +2502,26 @@ def complete_preflight_design(w, tmp_path): return design +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("fault", INVALID_SAVED_COSTS) +def test_preflight_refuses_invalid_saved_no_tools_cost_on_the_last_household( + world, tmp_path, monkeypatch, kind, fault +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + path = ( + Path(design["no_tools"]["runs"]) / cr.CHEAP[-1] + / "none/confirm/households" / f"{w.draw_order[-1]}.json" + ) + record = json.loads(path.read_text()) + invalidate_saved_cost(record, fault) + path.write_text(json.dumps(record)) + problems = cr.preflight(design, models) + assert problems and any("cost" in problem or "totals" in problem for problem in problems), ( + "preflight refuses invalid saved costs even on a model's final household" + ) + + def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( world, tmp_path, monkeypatch ): From c4f6a1085426b9c098e406bc7d8ccba54b9aa3a1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 17:18:32 -0500 Subject: [PATCH 10/31] Check development projections for each stage and its linked calculator before spending. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 124 +++++++++++++++-- tests/test_confirmatory_run.py | 241 ++++++++++++++++++++++++++++++--- 2 files changed, 338 insertions(+), 27 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 400abb1d..12366db0 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -352,6 +352,7 @@ def stage_for(design: dict) -> Stage: "preregistration", "frozen_sha256", "runs", + "development_runs", "stage", "board", "engine", @@ -982,9 +983,7 @@ class Ceiling(pilot.Spend): def __init__(self, ceiling_usd: float, spent_before: float): super().__init__(ceiling_usd, spent_before) - self.crossed_at: float | None = ( - clock() if spent_before >= ceiling_usd else None - ) + self.crossed_at: float | None = clock() if spent_before >= ceiling_usd else None self.closed = False # the records are final: spend is no longer counted self.unrecorded_usd = 0.0 # spend after that, in no record @@ -2626,6 +2625,11 @@ def web_preflight(design: dict) -> list[str]: f"linked calculator: {p}" for p in allocation_problems(calculator, STAGES["calculator"]) ) + if not allocation_problems(calculator, STAGES["calculator"]): + problems.extend( + f"linked calculator: {p}" + for p in projection_problems(calculator, STAGES["calculator"]) + ) for field in SHARED_STAGE_FIELDS: if field not in calculator or design.get(field) != calculator[field]: problems.append(f"web and calculator designs differ in {field}") @@ -2699,11 +2703,7 @@ def stage_start_problems(design: dict) -> list[str]: def allocation_problems(design: dict, stage: Stage) -> list[str]: - """Check the registered proportional cap split and three-times reserves. - - The independent report also compares expected costs with its development - projection. The runner has no access to that development evidence. - """ + """Check the registered proportional cap split and three-times reserves.""" problems = [] expected = {} for model in stage.models: @@ -2743,6 +2743,104 @@ def allocation_problems(design: dict, stage: Stage) -> list[str]: return problems +def registered_expected_costs(development_runs: Path, stage: Stage) -> dict[str, float]: + """Use exactly the report's development inventory and projection arithmetic. + + Calculator cheap models use all 320 finals in pe3/abl-* mctest folders; + frontier models use 10 pe/mctest finals. Web models use 100 finals per + arm across mctest and mafree, projecting the sum of the two arm means. + """ + from confirmatory_no_tools import read_regular, saved_children, saved_json_paths + + def costs(paths): + amounts = [] + for path in paths: + try: + record = json.loads(read_regular(path).decode("utf-8")) + if not isinstance(record, dict): + raise ValueError("development record must be an object") + amounts.append(attempt_cost(record)) + except ( + OSError, + ValueError, + TypeError, + AttributeError, + OverflowError, + ) as error: + raise ValueError(f"{path}: {error}") from error + return amounts + + expected = {} + for model in stage.models: + root = development_runs / model + if stage.kind == "web": + means = [] + for folder in ("web", "web-pe3"): + paths = [ + path + for board in ("mctest", "mafree") + for path in saved_json_paths(root / folder / board / "households") + ] + if len(paths) != 100: + raise ValueError( + f"{model}/{folder}: {len(paths)} development finals, " + "expected 100" + ) + means.append(math.fsum(costs(paths)) / len(paths)) + expected[model] = math.fsum(means) * HOUSEHOLDS + else: + arms = ( + sorted( + p + for p in saved_children(root) + if p.name == "pe3" or p.name.startswith("abl-") + ) + if model in CHEAP + else [root / "pe"] + ) + paths = [ + path + for arm in arms + for path in saved_json_paths(arm / "mctest" / "households") + ] + required = 320 if model in CHEAP else 10 + if len(paths) != required: + raise ValueError( + f"{model}: {len(paths)} development records for the cost " + f"projection, expected {required}" + ) + amounts = costs(paths) + expected[model] = ( + sum(amounts) + / len(amounts) + * HOUSEHOLDS + * (len(ARMS) if model in CHEAP else len(FRONTIER_ARMS)) + ) + return expected + + +def projection_problems(design: dict, stage: Stage) -> list[str]: + """Require declared expected costs to match the development projection.""" + try: + expected = registered_expected_costs(Path(design["development_runs"]), stage) + return [ + f"{model}: expected cost ${design['expected_costs'][model]}, " + f"and the development projection gives ${cost:.6f}" + for model, cost in expected.items() + if abs(float(design["expected_costs"][model]) - cost) > 1e-6 + ] + except ( + OSError, + KeyError, + ValueError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + ) as error: + return [f"the registered expected-cost projection: {error}"] + + def preflight(design: dict, models: tuple[str, ...]) -> list[str]: """What stops the run from starting.""" missing = [k for k in REQUIRED if k not in design] @@ -2755,10 +2853,16 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: if stage.kind == "calculator" and "frontier_arms" not in design: return ["the design has no frontier_arms"] problems = [] - if stage.kind == "calculator" and design.get("overall_cap") != CALCULATOR_OVERALL_CAP: + if ( + stage.kind == "calculator" + and design.get("overall_cap") != CALCULATOR_OVERALL_CAP + ): problems.append("the calculator design's overall_cap must be 150") problems.extend(stage_start_problems(design)) - problems.extend(allocation_problems(design, stage)) + allocation = allocation_problems(design, stage) + problems.extend(allocation) + if not allocation: + problems.extend(projection_problems(design, stage)) from confirmatory_no_tools import ( BOARD_INPUT_ERRORS, _board_input, diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index bbc103b6..0d87041d 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -996,6 +996,7 @@ def __init__( "preregistration": str(prereg), "frozen_sha256": frozen, "runs": str(tmp / "runs"), + "development_runs": str(tmp / "development-runs"), "stage": "confirm", "board": str(board_path), "engine": f"policyengine-us {ENGINE}", @@ -2402,7 +2403,14 @@ def test_the_preflight_names_a_missing_field(): INVALID_SAVED_COSTS = [ - "missing-totals", "missing-cost", None, False, True, "0", math.nan, -0.01, + "missing-totals", + "missing-cost", + None, + False, + True, + "0", + math.nan, + -0.01, ] @@ -2420,9 +2428,11 @@ def invalidate_saved_cost(record, fault): @pytest.mark.parametrize("fault", INVALID_SAVED_COSTS) def test_saved_attempt_cost_requires_an_explicit_valid_total(kind, record_kind, fault): record = { - "scenario_id": "last-household", "kind": kind, + "scenario_id": "last-household", + "kind": kind, "status": "harness_error" if record_kind == "stub" else "complete", - "attempts": [], "totals": {"cost_usd": 0.0}, + "attempts": [], + "totals": {"cost_usd": 0.0}, } assert cr.attempt_cost(record) == 0.0 invalidate_saved_cost(record, fault) @@ -2431,7 +2441,9 @@ def test_saved_attempt_cost_requires_an_explicit_valid_total(kind, record_kind, cr.attempt_cost(record) except ValueError: refused = True - assert refused, "saved attempt costs require a present nonboolean finite nonnegative number" + assert refused, ( + "saved attempt costs require a present nonboolean finite nonnegative number" + ) def test_live_attempt_cost_keeps_recording_a_missing_total(): @@ -2440,6 +2452,39 @@ def test_live_attempt_cost_keeps_recording_a_missing_total(): ) +def synthetic_development_tree(root, kind, expected): + """Only invented costs; retain the registered development folder counts.""" + stage = cr.STAGES[kind] + for model in stage.models: + if kind == "web": + unit = expected[model] / (2 * cr.HOUSEHOLDS) + groups = [ + (root / model / arm / board / "households", 50) + for arm in ("web", "web-pe3") + for board in ("mctest", "mafree") + ] + else: + unit = expected[model] / (cr.HOUSEHOLDS * len(stage.arms_for(model))) + arms = ( + ["pe3", *(f"abl-{n:02}" for n in range(15))] + if model in cr.CHEAP + else ["pe"] + ) + groups = [ + ( + root / model / arm / "mctest/households", + 20 if model in cr.CHEAP else 10, + ) + for arm in arms + ] + for folder, count in groups: + folder.mkdir(parents=True, exist_ok=True) + for n in range(count): + (folder / f"synthetic-{n:03}.json").write_text( + json.dumps({"totals": {"cost_usd": unit}}) + ) + + def complete_preflight_design(w, tmp_path): """Each calculator preflight starts after standing no-tools finals that pass the no-tools script's treatment checks: built through the harness from the @@ -2486,6 +2531,10 @@ def complete_preflight_design(w, tmp_path): "folder": "none", "stage": "confirm", }, + development_runs=str(tmp_path / "development-runs"), + ) + synthetic_development_tree( + Path(design["development_runs"]), "calculator", design["expected_costs"] ) for model in cr.CHEAP: for sid in design["draw_order"]: @@ -2510,16 +2559,18 @@ def test_preflight_refuses_invalid_saved_no_tools_cost_on_the_last_household( w, web = complete_web_design(world, tmp_path, monkeypatch) design, models = fresh_stage_design(w, web, tmp_path, kind) path = ( - Path(design["no_tools"]["runs"]) / cr.CHEAP[-1] - / "none/confirm/households" / f"{w.draw_order[-1]}.json" + Path(design["no_tools"]["runs"]) + / cr.CHEAP[-1] + / "none/confirm/households" + / f"{w.draw_order[-1]}.json" ) record = json.loads(path.read_text()) invalidate_saved_cost(record, fault) path.write_text(json.dumps(record)) problems = cr.preflight(design, models) - assert problems and any("cost" in problem or "totals" in problem for problem in problems), ( - "preflight refuses invalid saved costs even on a model's final household" - ) + assert problems and any( + "cost" in problem or "totals" in problem for problem in problems + ), "preflight refuses invalid saved costs even on a model's final household" def test_the_preflight_refuses_a_changed_draw_and_a_dirty_runner( @@ -2681,6 +2732,9 @@ def complete_web_design(world, tmp_path, monkeypatch, *, archives=False): ), ) del design["frontier_arms"] + synthetic_development_tree( + Path(design["development_runs"]), "web", design["expected_costs"] + ) monkeypatch.setattr( pilot, "_git", lambda *args: "0" * 40 if args[0] == "rev-parse" else "" ) @@ -2703,6 +2757,157 @@ def git_pins(args, *a, **kw): return w, design +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +def test_preflight_requires_expected_costs_to_match_the_development_projection( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + if kind == "linked": + design, models = copy.deepcopy(web), cr.WEB_MODELS + path = Path(design["calculator"]["path"]) + changed = json.loads(path.read_text()) + else: + design, models = fresh_stage_design(w, web, tmp_path, kind) + changed = design + for field in ("expected_costs", "reserves"): + changed[field] = {model: value * 2 for model, value in changed[field].items()} + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, models) + assert problems and all( + "development projection gives" in problem for problem in problems + ), "expected costs must match the registered development projection" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("fault", INVALID_SAVED_COSTS) +def test_development_projection_refuses_an_invalid_saved_attempt_cost( + world, tmp_path, monkeypatch, kind, fault +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + folder = "web" if kind == "web" else "pe3" + path = next((Path(design["development_runs"]) / models[0] / folder).rglob("*.json")) + record = json.loads(path.read_text()) + invalidate_saved_cost(record, fault) + path.write_text(json.dumps(record)) + problems = cr.preflight(design, models) + assert len(problems) == 1 and "expected-cost projection" in problems[0], ( + "development projection requires every saved attempt cost to be valid" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_requires_the_registered_record_count( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, kind) + folder = "web" if kind == "web" else "pe3" + path = next((Path(design["development_runs"]) / models[0] / folder).rglob("*.json")) + path.unlink() + problems = cr.preflight(design, models) + assert ( + len(problems) == 1 + and "development" in problems[0] + and "expected" in problems[0] + ), "development projection requires the registered final-record counts" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("direction", [-1, 1]) +def test_development_projection_accepts_its_tolerance_and_refuses_the_next_float( + world, tmp_path, monkeypatch, kind, direction +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + if kind == "linked": + design, models = copy.deepcopy(web), cr.WEB_MODELS + path = Path(design["calculator"]["path"]) + changed = json.loads(path.read_text()) + stage = cr.STAGES["calculator"] + else: + design, models = fresh_stage_design(w, web, tmp_path, kind) + changed, stage = design, cr.STAGES[kind] + model = stage.models[0] + projected = cr.registered_expected_costs(Path(changed["development_runs"]), stage)[ + model + ] + boundary = projected + direction * 1e-6 + # The greatest representable displacement that stays within the tolerance. + if abs(boundary - projected) > 1e-6: + boundary = math.nextafter(boundary, projected) + outward = math.inf if direction > 0 else -math.inf + for value, accepted in ( + (boundary, True), + (math.nextafter(boundary, outward), False), + ): + changed["expected_costs"][model] = value + total = math.fsum(changed["expected_costs"].values()) + share = cr.WEB_CAP if stage.kind == "web" else cr.CALCULATOR_CAP + changed["caps"] = { + m: share * c / total for m, c in changed["expected_costs"].items() + } + changed["reserves"] = { + m: 3 * c / cr.HOUSEHOLDS for m, c in changed["expected_costs"].items() + } + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, models) + assert (problems == []) is accepted, ( + "development projection uses the inclusive one-microdollar tolerance" + ) + if not accepted: + assert len(problems) == 1 and "development projection gives" in problems[0] + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_accepts_the_exact_one_microdollar_tolerance( + tmp_path, kind +): + stage = cr.STAGES[kind] + synthetic_development_tree(tmp_path, kind, {model: 1e-9 for model in stage.models}) + projected = cr.registered_expected_costs(tmp_path, stage) + model = stage.models[0] + boundary = projected[model] + 1e-6 + assert boundary - projected[model] == 1e-6 + design = {"development_runs": str(tmp_path), "expected_costs": dict(projected)} + design["expected_costs"][model] = boundary + assert cr.projection_problems(design, stage) == [], ( + "development projection accepts exactly its one-microdollar tolerance" + ) + design["expected_costs"][model] = math.nextafter(boundary, math.inf) + assert len(cr.projection_problems(design, stage)) == 1, ( + "development projection refuses one float beyond its one-microdollar tolerance" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_uses_the_registered_mean_and_arm_arithmetic( + tmp_path, kind +): + stage = cr.STAGES[kind] + expected = {model: 2.0 for model in stage.models} + synthetic_development_tree(tmp_path, kind, expected) + model = stage.models[0] + folder = "web" if kind == "web" else "pe3" + path = next((tmp_path / model / folder).rglob("*.json")) + record = json.loads(path.read_text()) + record["totals"]["cost_usd"] += 1.0 + path.write_text(json.dumps(record)) + got = cr.registered_expected_costs(tmp_path, stage) + increment = ( + cr.HOUSEHOLDS + / (100 if kind == "web" else 320) + * (1 if kind == "web" else len(cr.ARMS)) + ) + assert math.isclose(got[model], expected[model] + increment, abs_tol=1e-12), ( + "development projection preserves per-arm means " + "and registered household multipliers" + ) + + def fresh_stage_design(w, web, tmp_path, kind): """Keep every unrelated preflight input valid for an isolated budget fault.""" if kind == "web": @@ -2785,8 +2990,12 @@ def test_linked_calculator_design_refuses_a_start_at_its_175_dollar_ceiling( path.write_text(json.dumps(calculator)) design["calculator"]["sha256"] = cr.sha256_file(path) assert cr.preflight(design, cr.WEB_MODELS) == ( - ["linked calculator: spent_before must be finite and below the stage ceiling"] - if refused else [] + [ + "linked calculator: spent_before must be finite " + "and below the stage ceiling" + ] + if refused + else [] ), "a linked calculator design must have started below its $175 ceiling" @@ -3518,9 +3727,7 @@ def test_web_preflight_counts_all_attempts_stubs_and_late_spend_in_one_fsum( calculator = json.loads(Path(design["calculator"]["path"]).read_text()) calculator["spent_before"] = 0.0 Path(design["calculator"]["path"]).write_text(json.dumps(calculator)) - design["calculator"]["sha256"] = cr.sha256_file( - Path(design["calculator"]["path"]) - ) + design["calculator"]["sha256"] = cr.sha256_file(Path(design["calculator"]["path"])) # A flat sum retains tiny costs that per-model subtotals would round away. monkeypatch.setattr(cr, "calculator_evidence", lambda _: [1e16, 1.0, 1.0]) design["spent_before"] = 1e16 @@ -3656,9 +3863,9 @@ def test_web_execute_patches_gemini_before_its_first_completion( observed = [] def completion(**kwargs): - assert all(config.__dict__[name] is not value for name, value in original.items()), ( - "execute must install both Gemini patch methods before completion" - ) + assert all( + config.__dict__[name] is not value for name, value in original.items() + ), "execute must install both Gemini patch methods before completion" observed.append(True) def converse(lanes, sid, label, tool_call, spend): From 7026c62d8f77deba78b882659f6b01adadbf8ffb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 17:55:09 -0500 Subject: [PATCH 11/31] Check the linked calculator no-tools floor separately from the web floor. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 8 ++++++++ tests/test_confirmatory_run.py | 37 +++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 12366db0..9eb4faa4 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2654,6 +2654,7 @@ def web_preflight(design: dict) -> list[str]: try: calculator_costs = calculator_evidence(calculator) costs = list(calculator_costs) + no_tools_costs = [] spec = design["no_tools"] for model in saved_children(Path(spec["runs"])): if listed_is_dir(model): @@ -2664,9 +2665,16 @@ def web_preflight(design: dict) -> list[str]: / "households" ) costs.extend(saved) + no_tools_costs.extend(saved) calculator_spent_before = float(calculator["spent_before"]) if not math.isfinite(calculator_spent_before) or calculator_spent_before < 0: raise ValueError("calculator spent_before must be finite and nonnegative") + no_tools_spent = exact_spend(no_tools_costs) + if calculator_spent_before < no_tools_spent - 1e-6: + problems.append( + f"linked calculator spent_before ${calculator_spent_before} is " + f"below no-tools recorded spend ${no_tools_spent}" + ) spent = max( exact_spend([calculator_spent_before, *calculator_costs]), exact_spend(costs), diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0d87041d..0792f7c8 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2946,9 +2946,44 @@ def test_web_spent_before_covers_calculator_declaration_and_all_saved_costs( design["calculator"]["sha256"] = cr.sha256_file(path) design["spent_before"] = 88.0 problems = cr.preflight(design, cr.WEB_MODELS) - assert len(problems) == 1 and "recorded spend $89.0" in problems[0], ( + assert len(problems) == 2 and "recorded spend $89.0" in problems[1], ( "web spent_before also covers the complete saved-cost inventory" ) + assert "linked calculator spent_before" in problems[0], ( + "the linked calculator declaration independently covers saved no-tools costs" + ) + + +def test_linked_calculator_spent_before_covers_its_own_no_tools_floor( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["spent_before"] = 0.0 + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + final = next(Path(design["no_tools"]["runs"]).rglob("households/*.json")) + record = json.loads(final.read_text()) + record["totals"]["cost_usd"] = 1.0 + final.write_text(json.dumps(record)) + monkeypatch.setattr(cr, "calculator_evidence", lambda _: [88.0]) + design["spent_before"] = 89.0 + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and "linked calculator spent_before" in problems[0], ( + "a linked calculator declaration must cover its own saved no-tools floor" + ) + for amount, accepted in ( + (1.0 - 1e-6, True), + (math.nextafter(1.0 - 1e-6, -math.inf), False), + ): + calculator["spent_before"] = amount + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.preflight(design, cr.WEB_MODELS) + assert (problems == []) is accepted, ( + "the linked calculator floor uses the inclusive one-microdollar tolerance" + ) def test_calculator_preflight_requires_the_registered_150_dollar_cap( From 011dc6731713e6fb64002de61758e877d962c69b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 18:25:16 -0500 Subject: [PATCH 12/31] Validate design containers and money types while recording malformed inputs. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_no_tools.py | 5 +- scripts/confirmatory_run.py | 108 ++++++++++-- tests/test_confirmatory_input_reads.py | 15 +- tests/test_confirmatory_run.py | 217 ++++++++++++++++++++++++- 4 files changed, 318 insertions(+), 27 deletions(-) diff --git a/scripts/confirmatory_no_tools.py b/scripts/confirmatory_no_tools.py index 67a10b36..2260e283 100644 --- a/scripts/confirmatory_no_tools.py +++ b/scripts/confirmatory_no_tools.py @@ -661,6 +661,8 @@ def checked_board(): problems.append(f"no-tools {model}: {error}") spent = exact_spend(costs) try: + if type(design.get("spent_before")) not in (int, float): + raise ValueError("spent_before must be a nonboolean number") declared = float(design.get("spent_before")) except (TypeError, ValueError, OverflowError): declared = math.nan @@ -989,7 +991,8 @@ def no_tool(arguments): cls, _ = classify(record, 0.0, 0.0, None) outcome = action(cls, n) self.say( - f"{model} {sid} {n} {cls} {outcome} ${attempt_cost(record, saved=False):.6f}" + f"{model} {sid} {n} {cls} {outcome} " + f"${attempt_cost(record, saved=False):.6f}" ) if self.interrupted.is_set(): self.states[model] = "interrupted" diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 9eb4faa4..8eb1822d 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2594,6 +2594,25 @@ def finite(value): def web_preflight(design: dict) -> list[str]: + """Record malformed web designs as refusals before a provider can run.""" + try: + if not isinstance(design, dict): + raise ValueError("the design must be an object") + return _web_preflight(design) + except ( + OSError, + ValueError, + KeyError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + RuntimeError, + ) as error: + return [f"the web design is malformed: {error}"] + + +def _web_preflight(design: dict) -> list[str]: """The web stage follows its hashed, closed calculator stage and all spend.""" from confirmatory_no_tools import ( component, @@ -2621,11 +2640,9 @@ def web_preflight(design: dict) -> list[str]: if calculator.get("overall_cap") != CALCULATOR_OVERALL_CAP: problems.append("the linked calculator design's overall_cap must be 150") problems.extend(f"linked calculator: {p}" for p in stage_start_problems(calculator)) - problems.extend( - f"linked calculator: {p}" - for p in allocation_problems(calculator, STAGES["calculator"]) - ) - if not allocation_problems(calculator, STAGES["calculator"]): + allocation = allocation_problems(calculator, STAGES["calculator"]) + problems.extend(f"linked calculator: {p}" for p in allocation) + if not allocation: problems.extend( f"linked calculator: {p}" for p in projection_problems(calculator, STAGES["calculator"]) @@ -2666,6 +2683,8 @@ def web_preflight(design: dict) -> list[str]: ) costs.extend(saved) no_tools_costs.extend(saved) + if type(calculator["spent_before"]) not in (int, float): + raise ValueError("calculator spent_before must be a nonboolean number") calculator_spent_before = float(calculator["spent_before"]) if not math.isfinite(calculator_spent_before) or calculator_spent_before < 0: raise ValueError("calculator spent_before must be finite and nonnegative") @@ -2679,6 +2698,8 @@ def web_preflight(design: dict) -> list[str]: exact_spend([calculator_spent_before, *calculator_costs]), exact_spend(costs), ) + if type(design["spent_before"]) not in (int, float): + raise ValueError("web spent_before must be a nonboolean number") declared = float(design["spent_before"]) if not 0 <= declared < math.inf or declared < spent - 1e-6: problems.append( @@ -2701,9 +2722,19 @@ def web_preflight(design: dict) -> list[str]: def stage_start_problems(design: dict) -> list[str]: """No provider request may start once the whole-run ceiling is reached.""" try: + if any( + type(design[field]) not in (int, float) + for field in ("spent_before", "overall_cap") + ): + raise ValueError("stage money must be nonboolean numbers") spent_before = float(design["spent_before"]) - ceiling = float(design["overall_cap"]) + CEILING_MARGIN - if math.isfinite(spent_before) and spent_before < ceiling: + overall_cap = float(design["overall_cap"]) + ceiling = overall_cap + CEILING_MARGIN + if ( + math.isfinite(overall_cap) + and math.isfinite(spent_before) + and 0 <= spent_before < ceiling + ): return [] except (KeyError, TypeError, ValueError, OverflowError): pass @@ -2713,12 +2744,24 @@ def stage_start_problems(design: dict) -> list[str]: def allocation_problems(design: dict, stage: Stage) -> list[str]: """Check the registered proportional cap split and three-times reserves.""" problems = [] + for field in ("expected_costs", "caps", "reserves"): + values = design.get(field) + if not isinstance(values, dict) or set(values) != set(stage.models): + problems.append( + f"{field} must be an object with exactly the registered model keys" + ) + if problems: + return problems expected = {} for model in stage.models: try: - value = design["expected_costs"][model] - cost = float(value) - if isinstance(value, bool) or not math.isfinite(cost) or cost <= 0: + raw_cost = design["expected_costs"][model] + cost = float(raw_cost) + if ( + type(raw_cost) not in (int, float) + or not math.isfinite(cost) + or cost <= 0 + ): raise ValueError("expected cost is not finite and positive") expected[model] = cost except (KeyError, TypeError, ValueError, OverflowError): @@ -2738,9 +2781,9 @@ def allocation_problems(design: dict, stage: Stage) -> list[str]: ("reserves", 3 * cost / HOUSEHOLDS, RESERVE_TOLERANCE, "rule"), ): try: - value = design[field][model] - got = float(value) - if isinstance(value, bool) or not math.isfinite(got): + raw_allocation = design[field][model] + got = float(raw_allocation) + if type(raw_allocation) not in (int, float) or not math.isfinite(got): raise ValueError("allocation is not finite") except (KeyError, TypeError, ValueError, OverflowError): problems.append(f"{model}: {field} needs a finite allocation") @@ -2758,7 +2801,19 @@ def registered_expected_costs(development_runs: Path, stage: Stage) -> dict[str, frontier models use 10 pe/mctest finals. Web models use 100 finals per arm across mctest and mafree, projecting the sum of the two arm means. """ - from confirmatory_no_tools import read_regular, saved_children, saved_json_paths + from confirmatory_no_tools import read_regular, saved_children + + def development_paths(folder): + # Match the report's projection-only empty-inventory semantics. + try: + with os.scandir(folder) as entries: + return sorted( + Path(entry.path) + for entry in entries + if entry.name.endswith(".json") + ) + except (FileNotFoundError, NotADirectoryError): + return [] def costs(paths): amounts = [] @@ -2787,7 +2842,7 @@ def costs(paths): paths = [ path for board in ("mctest", "mafree") - for path in saved_json_paths(root / folder / board / "households") + for path in development_paths(root / folder / board / "households") ] if len(paths) != 100: raise ValueError( @@ -2809,7 +2864,7 @@ def costs(paths): paths = [ path for arm in arms - for path in saved_json_paths(arm / "mctest" / "households") + for path in development_paths(arm / "mctest" / "households") ] required = 320 if model in CHEAP else 10 if len(paths) != required: @@ -2850,6 +2905,25 @@ def projection_problems(design: dict, stage: Stage) -> list[str]: def preflight(design: dict, models: tuple[str, ...]) -> list[str]: + """Return recorded input problems for every malformed design.""" + try: + if not isinstance(design, dict): + raise ValueError("the design must be an object") + return _preflight(design, models) + except ( + OSError, + ValueError, + KeyError, + TypeError, + AttributeError, + RecursionError, + OverflowError, + RuntimeError, + ) as error: + return [f"the design is malformed: {error}"] + + +def _preflight(design: dict, models: tuple[str, ...]) -> list[str]: """What stops the run from starting.""" missing = [k for k in REQUIRED if k not in design] if missing: @@ -3009,8 +3083,6 @@ def household_paths(stage): return saved_json_paths(stage / "households") for model in models: - if model not in design["caps"] or model not in design["reserves"]: - problems.append(f"{model}: no cap or reserve") for label in stage.arms_for(model): try: arm_profile(model, label) diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index e85e7562..c7bd6f1a 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -425,20 +425,21 @@ def test_preflight_refuses_a_design_without_spent_before(tmp_path, monkeypatch): @pytest.mark.parametrize( - "declared, refused", - [("1", False), ("0.1", True), (True, False), (False, True)], + "declared", + ["1", "0.1", True, False], ids=["text-enough", "text-short", "true", "false"], ) -def test_preflight_counts_numeric_text_and_booleans_as_numbers( - tmp_path, monkeypatch, declared, refused +def test_preflight_refuses_numeric_text_and_booleans_as_spend( + tmp_path, monkeypatch, declared ): - """As the report does: float() decides what a spent_before is worth.""" + """A declaration must be a nonboolean JSON number before conversion.""" world, design = pool_design(tmp_path, monkeypatch) saved_spend(world, design) design["spent_before"] = declared problems = cr.preflight(design, (cr.CHEAP[0],)) - assert any(p.startswith("no-tools spend") for p in problems) is refused, problems - assert not any("finite, nonnegative" in p for p in problems), problems + assert any("finite, nonnegative" in p for p in problems), ( + "saved no-tools spend refuses nonnumeric or boolean declarations" + ) @pytest.mark.parametrize( diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0792f7c8..eacf1a8f 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2883,6 +2883,65 @@ def test_development_projection_accepts_the_exact_one_microdollar_tolerance( ) +def test_development_projection_ignores_a_non_directory_unused_arm(tmp_path): + stage = cr.STAGES["calculator"] + synthetic_development_tree( + tmp_path, "calculator", {model: 1.0 for model in stage.models} + ) + before = cr.registered_expected_costs(tmp_path, stage) + (tmp_path / stage.models[0] / "abl-unused").write_text("unused") + raised, after = None, None + try: + after = cr.registered_expected_costs(tmp_path, stage) + except (OSError, ValueError) as error: + raised = error + assert raised is None and after == before, ( + "development projection preserves the report's empty non-directory inventories" + ) + + +def test_web_development_projection_counts_the_other_board_when_one_is_a_file(tmp_path): + stage = cr.STAGES["web"] + synthetic_development_tree(tmp_path, "web", {model: 1.0 for model in stage.models}) + before = cr.registered_expected_costs(tmp_path, stage) + arm = tmp_path / stage.models[0] / "web" + mctest = arm / "mctest" + for path in (mctest / "households").iterdir(): + path.rename(arm / "mafree/households" / f"moved-{path.name}") + (mctest / "households").rmdir() + mctest.rmdir() + mctest.write_text("unused") + raised, after = None, None + try: + after = cr.registered_expected_costs(tmp_path, stage) + except (OSError, ValueError) as error: + raised = error + assert raised is None and after == before, ( + "web development projection preserves the report's " + "empty non-directory inventories" + ) + + +@pytest.mark.parametrize("linked", [False, True]) +def test_web_preflight_spend_declarations_require_nonboolean_numbers( + world, tmp_path, monkeypatch, linked +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + target = calculator if linked else design + original = target["spent_before"] + for value in (str(original), False): + target["spent_before"] = value + if linked: + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + problems = cr.web_preflight(design) + assert any("must be a nonboolean number" in problem for problem in problems), ( + "web floor declarations require nonboolean numbers before conversion" + ) + + @pytest.mark.parametrize("kind", ["calculator", "web"]) def test_development_projection_uses_the_registered_mean_and_arm_arithmetic( tmp_path, kind @@ -2986,6 +3045,160 @@ def test_linked_calculator_spent_before_covers_its_own_no_tools_floor( ) +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["expected_costs", "caps", "reserves"]) +def test_design_money_containers_have_exactly_the_registered_model_keys( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + original = ( + json.loads(path.read_text()) if kind == "linked" else copy.deepcopy(design) + ) + for fault in (None, [], "invalid", "extra-good", "extra-bad"): + changed = copy.deepcopy(original) + if isinstance(fault, str) and fault.startswith("extra-"): + changed[field]["unused"] = 1.0 if fault == "extra-good" else "bad" + else: + changed[field] = fault + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + else: + design = changed + problems = cr.preflight(design, models) + prefix = "linked calculator: " if kind == "linked" else "" + assert problems == [ + prefix + f"{field} must be an object with exactly the registered model keys" + ], "money containers must be objects with exactly the registered model keys" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["expected_costs", "caps", "reserves"]) +def test_design_money_values_are_numbers_before_conversion( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + original = ( + json.loads(path.read_text()) if kind == "linked" else copy.deepcopy(design) + ) + model = cr.STAGES["calculator" if kind == "linked" else kind].models[0] + for value in (str(original[field][model]), True): + changed = copy.deepcopy(original) + changed[field][model] = value + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + else: + design = changed + problems = cr.preflight(design, models) + assert ( + len(problems) == 1 + and ( + "finite and positive" + if field == "expected_costs" + else "finite allocation" + ) + in problems[0] + ), "money values must be nonboolean numbers before conversion" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["overall_cap", "spent_before"]) +def test_design_money_scalars_refuse_malformed_values_without_raising( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + original = ( + json.loads(path.read_text()) if kind == "linked" else copy.deepcopy(design) + ) + for value in (10**400, str(original[field]), None, True, math.nan): + changed = copy.deepcopy(original) + changed[field] = value + if kind == "linked": + path.write_text(json.dumps(changed)) + design["calculator"]["sha256"] = cr.sha256_file(path) + else: + design = changed + raised, problems = None, [] + try: + problems = cr.preflight(design, models) + if kind == "web": + assert cr.web_preflight(design) + except (TypeError, ValueError, OverflowError, AttributeError) as error: + raised = error + assert raised is None and problems, ( + "malformed monetary scalars must produce recorded refusals without raising" + ) + + +@pytest.mark.parametrize("field", ["overall_cap", "spent_before"]) +def test_stage_money_scalars_require_nonboolean_numbers(field): + for value in ("1", True, False): + design = {"overall_cap": 150.0, "spent_before": 1.0} + design[field] = value + assert cr.stage_start_problems(design), ( + "stage monetary scalars require nonboolean numbers before conversion" + ) + + +def test_stage_money_scalars_are_finite_and_prior_spend_is_nonnegative(): + for field, value in ( + ("overall_cap", math.inf), + ("overall_cap", math.nan), + ("spent_before", math.inf), + ("spent_before", math.nan), + ("spent_before", -0.01), + ): + design = {"overall_cap": 150.0, "spent_before": 1.0} + design[field] = value + assert cr.stage_start_problems(design), ( + "stage money must be finite and prior spend must be nonnegative" + ) + + +@pytest.mark.parametrize("check", ["preflight", "web_preflight"]) +def test_malformed_design_objects_become_recorded_problems(check): + for design in (None, [], "invalid", True): + raised, problems = None, [] + try: + problems = ( + cr.preflight(design, cr.WEB_MODELS) + if check == "preflight" + else cr.web_preflight(design) + ) + except (TypeError, ValueError, OverflowError, AttributeError) as error: + raised = error + assert raised is None and len(problems) == 1 and "malformed" in problems[0], ( + "malformed design objects must become recorded problems without raising" + ) + + +def test_malformed_design_paths_become_recorded_problems(world, tmp_path): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + design["preregistration"] = None + raised, problems = None, [] + try: + problems = cr.preflight(design, (cr.CHEAP[0],)) + except (TypeError, ValueError, OverflowError, AttributeError) as error: + raised = error + assert raised is None and len(problems) == 1 and "malformed" in problems[0], ( + "malformed design paths must become recorded problems without raising" + ) + + def test_calculator_preflight_requires_the_registered_150_dollar_cap( world, tmp_path, monkeypatch ): @@ -3103,7 +3316,9 @@ def test_stage_allocation_requires_each_expected_cost_to_be_finite_and_positive( for m, value in design["expected_costs"].items() } assert cr.preflight(design, models) == [ - f"{model}: expected_costs must be finite and positive" + "expected_costs must be an object with exactly the registered model keys" + if cost is None + else f"{model}: expected_costs must be finite and positive" ], "each stage model needs a finite positive expected cost" From e6ffd17b5b4e681d2f3825bc26d10608f315cf06 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 4 Nov 2026 18:28:40 -0500 Subject: [PATCH 13/31] Isolate allocation guards and pin tolerance and flat-sum boundaries. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 2 - tests/test_confirmatory_input_reads.py | 5 +- tests/test_confirmatory_run.py | 89 ++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 8eb1822d..3dd52222 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2820,8 +2820,6 @@ def costs(paths): for path in paths: try: record = json.loads(read_regular(path).decode("utf-8")) - if not isinstance(record, dict): - raise ValueError("development record must be an object") amounts.append(attempt_cost(record)) except ( OSError, diff --git a/tests/test_confirmatory_input_reads.py b/tests/test_confirmatory_input_reads.py index c7bd6f1a..df9994f9 100644 --- a/tests/test_confirmatory_input_reads.py +++ b/tests/test_confirmatory_input_reads.py @@ -435,9 +435,10 @@ def test_preflight_refuses_numeric_text_and_booleans_as_spend( """A declaration must be a nonboolean JSON number before conversion.""" world, design = pool_design(tmp_path, monkeypatch) saved_spend(world, design) + assert nt.stage_problems(design) == [] design["spent_before"] = declared - problems = cr.preflight(design, (cr.CHEAP[0],)) - assert any("finite, nonnegative" in p for p in problems), ( + problems = nt.stage_problems(design) + assert len(problems) == 1 and "finite, nonnegative" in problems[0], ( "saved no-tools spend refuses nonnumeric or boolean declarations" ) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index eacf1a8f..0cfc212f 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -3363,6 +3363,95 @@ def test_stage_allocation_accepts_the_reports_cap_and_reserve_tolerances( ) +def allocation_guard_design(kind, *, target_cost=1.0): + stage = cr.STAGES[kind] + expected = {model: 1.0 for model in stage.models} + expected[stage.models[0]] = target_cost + return allocated_design(stage, expected) + + +def allocated_design(stage, expected): + total = math.fsum(expected.values()) + share = cr.WEB_CAP if stage.kind == "web" else cr.CALCULATOR_CAP + return { + "kind": stage.kind, + "expected_costs": expected, + "caps": {model: share * cost / total for model, cost in expected.items()}, + "reserves": { + model: 3 * cost / cr.HOUSEHOLDS for model, cost in expected.items() + }, + } + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("field", ["caps", "reserves"]) +@pytest.mark.parametrize("value", [math.nan, math.inf, -math.inf, True, False]) +def test_each_stage_allocation_is_finite_and_nonboolean_alone(kind, field, value): + stage = cr.STAGES[kind] + model = stage.models[0] + design = allocation_guard_design(kind) + if value is True: + if field == "caps": + share = cr.WEB_CAP if kind == "web" else cr.CALCULATOR_CAP + expected = {m: (share - 1) / (len(stage.models) - 1) for m in stage.models} + expected[model] = 1.0 + design = allocated_design(stage, expected) + else: + design = allocation_guard_design(kind, target_cost=cr.HOUSEHOLDS / 3) + elif value is False: + design = allocation_guard_design(kind, target_cost=1e-12) + assert cr.allocation_problems(design, stage) == [] + if isinstance(value, bool): + tolerance = 0.005 if field == "caps" else 0.0005 + assert abs(float(value) - design[field][model]) <= tolerance + design[field][model] = value + assert cr.allocation_problems(design, stage) == [ + f"{model}: {field} needs a finite allocation" + ], "each allocation must be finite and nonboolean even when its formula agrees" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("field", ["caps", "reserves"]) +@pytest.mark.parametrize("side", [-1, 1]) +def test_each_stage_allocation_accepts_exact_tolerance_and_refuses_next_float( + kind, field, side +): + stage = cr.STAGES[kind] + model = stage.models[0] + design = allocation_guard_design(kind, target_cost=1e-9) + want = design[field][model] + tolerance = 0.005 if field == "caps" else 0.0005 + boundary = want + side * tolerance + assert abs(boundary - want) == tolerance + design[field][model] = boundary + assert cr.allocation_problems(design, stage) == [], ( + "allocation formulas include their exact tolerance on both sides" + ) + design[field][model] = math.nextafter(boundary, math.inf if side > 0 else -math.inf) + problems = cr.allocation_problems(design, stage) + assert len(problems) == 1 and f"{model}: " in problems[0], ( + "allocation formulas refuse one representable step beyond either tolerance" + ) + + +def test_web_declared_calculator_floor_uses_one_flat_fsum(world, tmp_path, monkeypatch): + _, design = complete_web_design(world, tmp_path, monkeypatch) + monkeypatch.setattr(cr, "CEILING_MARGIN", 1e17) + costs, declared_prior = [1e16, 1.0], 1.0 + calculator = json.loads(Path(design["calculator"]["path"]).read_text()) + assert calculator["spent_before"] == declared_prior + assert declared_prior + math.fsum(costs) == 1e16 + assert math.fsum([declared_prior, *costs]) == 1e16 + 2 + monkeypatch.setattr(cr, "calculator_evidence", lambda _: costs) + design["spent_before"] = 1e16 + problems = cr.preflight(design, cr.WEB_MODELS) + assert len(problems) == 1 and "recorded spend" in problems[0], ( + "the calculator declaration and all calculator costs require one flat fsum" + ) + design["spent_before"] = math.fsum([declared_prior, *costs]) + assert cr.preflight(design, cr.WEB_MODELS) == [] + + @pytest.mark.parametrize("kind", ["calculator", "web"]) def test_stage_allocation_refuses_an_overflowing_expected_cost_total(kind): stage = cr.STAGES[kind] From 6d25c67707761317d77f7040ef9b38ed8a7363db Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 06:03:40 -0500 Subject: [PATCH 14/31] Reject rehearsal options and bind stages to one development root. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 14 ++++++++++-- tests/test_confirmatory_run.py | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 3dd52222..ae734035 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -234,6 +234,7 @@ def folder_key_for(self, model: str) -> str: "run_commit", "workers", "no_tools", + "development_runs", ) WEB_SCORING_FIELDS = ( "build", @@ -2639,6 +2640,7 @@ def _web_preflight(design: dict) -> list[str]: return [*problems, f"the calculator design cannot be read: {error}"] if calculator.get("overall_cap") != CALCULATOR_OVERALL_CAP: problems.append("the linked calculator design's overall_cap must be 150") + problems.extend(f"linked calculator: {p}" for p in rehearsal_problems(calculator)) problems.extend(f"linked calculator: {p}" for p in stage_start_problems(calculator)) allocation = allocation_problems(calculator, STAGES["calculator"]) problems.extend(f"linked calculator: {p}" for p in allocation) @@ -2719,6 +2721,15 @@ def _web_preflight(design: dict) -> list[str]: return problems +def rehearsal_problems(design: dict) -> list[str]: + """Rehearsal-only options cannot be present in a frozen design.""" + return [ + f"the confirmatory design cannot contain the rehearsal option {field}" + for field in ("schema_hash_exempt", "stated_engine_overrides") + if field in design + ] + + def stage_start_problems(design: dict) -> list[str]: """No provider request may start once the whole-run ceiling is reached.""" try: @@ -2982,8 +2993,7 @@ def _preflight(design: dict, models: tuple[str, ...]) -> list[str]: or shutil.which(str(command[0])) is None ): problems.append(f"the anchor command {command!r} is not a command here") - if design.get("stated_engine_overrides"): - problems.append("the confirmatory run states the board's engine: no overrides") + problems.extend(rehearsal_problems(design)) preregistration = Path(design["preregistration"]) try: if ( diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0cfc212f..ad6f5d82 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5272,3 +5272,43 @@ def test_the_treatment_code_is_the_treatment_commits(): assert changed == 0, ( "move TREATMENT_COMMIT to the last commit that touches TREATMENT_PATHS" ) + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize( + "field,value", [("schema_hash_exempt", []), ("stated_engine_overrides", {})] +) +def test_preflight_refuses_the_presence_of_each_rehearsal_option( + world, tmp_path, monkeypatch, kind, field, value +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + target = json.loads(path.read_text()) if kind == "linked" else design + target[field] = value + if kind == "linked": + path.write_text(json.dumps(target)) + design["calculator"]["sha256"] = cr.sha256_file(path) + prefix = "linked calculator: " if kind == "linked" else "" + assert cr.preflight(design, models) == [ + prefix + f"the confirmatory design cannot contain the rehearsal option {field}" + ], "preflight refuses rehearsal-option presence even when its value is empty" + + +def test_web_preflight_refuses_a_linked_development_root_difference_alone( + world, tmp_path, monkeypatch +): + _, design = complete_web_design(world, tmp_path, monkeypatch) + path = Path(design["calculator"]["path"]) + calculator = json.loads(path.read_text()) + calculator["development_runs"] = str(tmp_path / "other-development-runs") + synthetic_development_tree( + Path(calculator["development_runs"]), "calculator", calculator["expected_costs"] + ) + path.write_text(json.dumps(calculator)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, cr.WEB_MODELS) == [ + "web and calculator designs differ in development_runs" + ], "linked calculator and web designs must share their development root" From ac079e731263235304147536bbd3e0cb2f5c27ea Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 06:07:08 -0500 Subject: [PATCH 15/31] Require nonnegative allocations within the existing formula tolerances. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 3 +++ tests/test_confirmatory_run.py | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index ae734035..68841b38 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -2799,6 +2799,9 @@ def allocation_problems(design: dict, stage: Stage) -> list[str]: except (KeyError, TypeError, ValueError, OverflowError): problems.append(f"{model}: {field} needs a finite allocation") continue + if got < 0: + problems.append(f"{model}: {field} needs a nonnegative allocation") + continue if abs(got - want) > tolerance: name = "cap" if field == "caps" else "reserve" problems.append(f"{model}: {name} ${got}, and the {rule} gives ${want}") diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index ad6f5d82..2617aef8 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5312,3 +5312,40 @@ def test_web_preflight_refuses_a_linked_development_root_difference_alone( assert cr.preflight(design, cr.WEB_MODELS) == [ "web and calculator designs differ in development_runs" ], "linked calculator and web designs must share their development root" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +@pytest.mark.parametrize("field", ["caps", "reserves"]) +def test_preflight_refuses_negative_allocations_even_inside_formula_tolerances( + world, tmp_path, monkeypatch, kind, field +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) + path = Path(web["calculator"]["path"]) + target = json.loads(path.read_text()) if kind == "linked" else design + stage = cr.STAGES["calculator" if kind == "linked" else kind] + model = stage.models[0] + expected = {m: 1.0 for m in stage.models} + expected[model] = 0.0 + synthetic_development_tree(Path(target["development_runs"]), stage.kind, expected) + expected[model] = 1e-20 + target["expected_costs"] = expected + total = math.fsum(expected.values()) + share = 120 if stage.kind == "web" else 145 + target["caps"] = {m: share * c / total for m, c in expected.items()} + target["reserves"] = {m: 3 * c / 120 for m, c in expected.items()} + prefix = "linked calculator: " if kind == "linked" else "" + for value, want in ( + (0.0, []), + (-0.0, []), + (-0.0001, [prefix + f"{model}: {field} needs a nonnegative allocation"]), + ): + target[field][model] = value + if kind == "linked": + path.write_text(json.dumps(target)) + design["calculator"]["sha256"] = cr.sha256_file(path) + assert cr.preflight(design, models) == want, ( + "allocations must be nonnegative even inside their formula tolerance; signed zero is allowed" + ) From 9d8ff8a5b591130d9d54fea387fa1b870267889b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 11:43:27 -0500 Subject: [PATCH 16/31] Report malformed command inputs before reading dependent design fields. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 20 +++++++-- tests/test_confirmatory_run.py | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 68841b38..f0b174b8 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -3146,15 +3146,24 @@ def main(argv=None) -> int: resume.add_argument("model") resume.add_argument("--note", default="the account was fixed") args = parser.parse_args(argv) - design = json.loads(args.design.read_text()) try: + design = json.loads(args.design.read_text()) + if not isinstance(design, dict): + raise ValueError("the design must be an object") stage = stage_for(design) - except ValueError as error: + except (OSError, ValueError, TypeError, AttributeError, RecursionError) as error: print(f"The run cannot start: {error}") return 1 if args.command == "resume": - folder = Path(design["runs"]) / args.model - write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) + try: + runs = design.get("runs") + if not isinstance(runs, str) or not runs.strip(): + raise ValueError("the design must name a usable runs folder") + folder = Path(runs) / args.model + write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) + except (OSError, ValueError, TypeError) as error: + print(f"The run cannot start: {error}") + return 1 print(f"{args.model}: resume requested; the runner resumes within a second") return 0 if args.workers is not None and args.workers != design.get("workers"): @@ -3165,6 +3174,9 @@ def main(argv=None) -> int: return 1 models = stage.models problems = preflight(design, models) + if problems: + print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) + return 1 from confirmatory_no_tools import BOARD_INPUT_ERRORS board = Path(design["board"]) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 2617aef8..2ba23f44 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5349,3 +5349,77 @@ def test_preflight_refuses_negative_allocations_even_inside_formula_tolerances( assert cr.preflight(design, models) == want, ( "allocations must be nonnegative even inside their formula tolerance; signed zero is allowed" ) + + +@pytest.mark.parametrize("command", ["run", "resume"]) +@pytest.mark.parametrize("fault", ["missing", "invalid-json", "null", "list"]) +def test_command_line_reports_unreadable_or_nonobject_designs( + tmp_path, capsys, command, fault +): + path = tmp_path / "design.json" + if fault != "missing": + path.write_text({"invalid-json": "{", "null": "null", "list": "[]"}[fault]) + argv = [command, str(path)] + (["gpt-6-luna"] if command == "resume" else []) + raised = None + try: + result = cr.main(argv) + except Exception as error: + raised = error + result = None + assert ( + raised is None + and result == 1 + and capsys.readouterr().out.startswith("The run cannot start: ") + ), ( + "the command line reports unreadable JSON and non-object designs as startup refusals" + ) + + +@pytest.mark.parametrize("board", ["missing", None]) +def test_command_line_prints_preflight_refusals_before_reading_the_board( + tmp_path, capsys, monkeypatch, board +): + path = tmp_path / "design.json" + design = {"kind": "calculator"} + if board is None: + design["board"] = None + path.write_text(json.dumps(design)) + monkeypatch.setattr( + pilot, + "_require_engine", + lambda _: pytest.fail("engine check must follow a successful preflight"), + ) + raised = None + try: + result = cr.main(["run", str(path)]) + except Exception as error: + raised = error + result = None + assert ( + raised is None + and result == 1 + and "The run cannot start:\n- the design has no" in capsys.readouterr().out + ), "the command line prints preflight refusals before dependent board reads" + + +@pytest.mark.parametrize("runs", ["missing", None, "", [], {}]) +def test_command_line_resume_refuses_a_design_without_usable_runs( + tmp_path, capsys, runs +): + path = tmp_path / "design.json" + design = {"kind": "calculator"} + if runs != "missing": + design["runs"] = runs + path.write_text(json.dumps(design)) + raised = None + try: + result = cr.main(["resume", str(path), "gpt-6-luna"]) + except Exception as error: + raised = error + result = None + assert ( + raised is None + and result == 1 + and capsys.readouterr().out.startswith("The run cannot start: ") + ), "resume reports missing or unusable runs folders as startup refusals" + From d37f68f1dfc3f7b19e6d253ead590e3c7caaa47e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 13:37:11 -0500 Subject: [PATCH 17/31] Keep allocation boundary fixtures valid under the nonnegative sign rule. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 2ba23f44..8516da19 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -3418,9 +3418,21 @@ def test_each_stage_allocation_accepts_exact_tolerance_and_refuses_next_float( ): stage = cr.STAGES[kind] model = stage.models[0] - design = allocation_guard_design(kind, target_cost=1e-9) - want = design[field][model] tolerance = 0.005 if field == "caps" else 0.0005 + if field == "caps": + share = 120 if kind == "web" else 145 + expected = { + m: (share - tolerance) / (len(stage.models) - 1) + for m in stage.models + } + expected[model] = tolerance + design = allocated_design(stage, expected) + else: + design = allocation_guard_design( + kind, target_cost=tolerance * cr.HOUSEHOLDS / 3 + ) + want = design[field][model] + assert want == tolerance boundary = want + side * tolerance assert abs(boundary - want) == tolerance design[field][model] = boundary @@ -5335,7 +5347,7 @@ def test_preflight_refuses_negative_allocations_even_inside_formula_tolerances( total = math.fsum(expected.values()) share = 120 if stage.kind == "web" else 145 target["caps"] = {m: share * c / total for m, c in expected.items()} - target["reserves"] = {m: 3 * c / 120 for m, c in expected.items()} + target["reserves"] = {m: 3 * c / cr.HOUSEHOLDS for m, c in expected.items()} prefix = "linked calculator: " if kind == "linked" else "" for value, want in ( (0.0, []), From b9ee484db5ea0a63539cd00911dd4d0dc0489553 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 13:37:40 -0500 Subject: [PATCH 18/31] Prove both development projection readers refuse FIFOs without blocking. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 8516da19..50addf3c 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5435,3 +5435,46 @@ def test_command_line_resume_refuses_a_design_without_usable_runs( and capsys.readouterr().out.startswith("The run cannot start: ") ), "resume reports missing or unusable runs folders as startup refusals" +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_development_projection_refuses_a_fifo_without_blocking(tmp_path, kind): + stage = cr.STAGES[kind] + synthetic_development_tree(tmp_path, kind, {model: 1.0 for model in stage.models}) + arm = "web" if kind == "web" else "pe3" + path = next((tmp_path / stage.models[0] / arm).rglob("*.json")) + path.unlink() + os.mkfifo(path) + done = threading.Event() + outcome = [] + + def read_projection(): + try: + outcome.append(cr.registered_expected_costs(tmp_path, stage)) + except Exception as error: + outcome.append(error) + finally: + done.set() + + reader = threading.Thread(target=read_projection, daemon=True) + reader.start() + finished = done.wait(2) + if not finished: + # Release an ordinary blocking open in a reverted reader, so even a + # failing guard test leaves no stuck reader or pytest process behind. + try: + descriptor = os.open(path, os.O_WRONLY | os.O_NONBLOCK) + except OSError: + descriptor = None + if descriptor is not None: + try: + os.write(descriptor, b'{"totals":{"cost_usd":0}}') + finally: + os.close(descriptor) + reader.join(2) + assert ( + finished + and len(outcome) == 1 + and isinstance(outcome[0], ValueError) + and str(path) in str(outcome[0]) + and "FIFO" in str(outcome[0]) + ), "development record reads refuse FIFO inputs promptly with a path-bearing error" + From d73007c95391e90f4924e45647166cdad73ffdcf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 13:38:03 -0500 Subject: [PATCH 19/31] Check every development inventory count and ignore populated unregistered paths. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 72 ++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 50addf3c..c15bed70 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5478,3 +5478,75 @@ def read_projection(): and "FIFO" in str(outcome[0]) ), "development record reads refuse FIFO inputs promptly with a path-bearing error" +@pytest.mark.parametrize( + "inventory,count,linked", + [ + (inventory, count, linked) + for inventory, counts in (("cheap", (319, 321)), ("frontier", (9, 11))) + for count in counts + for linked in (False, True) + ] + + [ + (inventory, count, False) + for inventory in ("web", "web-pe3") + for count in (99, 101) + ], + ids=[ + f"{inventory}-{count}-{'linked' if linked else 'direct'}" + for inventory, counts in (("cheap", (319, 321)), ("frontier", (9, 11))) + for count in counts + for linked in (False, True) + ] + + [f"{inventory}-{count}-direct" for inventory in ("web", "web-pe3") for count in (99, 101)], +) +def test_projection_refuses_missing_and_extra_finals_for_each_inventory( + world, tmp_path, monkeypatch, inventory, count, linked +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + kind = "web" if inventory.startswith("web") else "calculator" + design, models = fresh_stage_design(w, web, tmp_path, "web" if linked else kind) + if inventory == "cheap": + model, arm, required = cr.CHEAP[0], "pe3", 320 + elif inventory == "frontier": + model, arm, required = cr.FRONTIER[0], "pe", 10 + else: + model, arm, required = cr.WEB_MODELS[0], inventory, 100 + folder = Path(design["development_runs"]) / model / arm / "mctest/households" + if count < required: + next(folder.glob("*.json")).unlink() + else: + source = next(folder.glob("*.json")) + (folder / "extra-final.json").write_bytes(source.read_bytes()) + problems = cr.preflight(design, models) + assert ( + len(problems) == 1 + and "expected-cost projection" in problems[0] + and f"expected {required}" in problems[0] + and str(count) in problems[0] + and (not linked or problems[0].startswith("linked calculator: ")) + ), "development projection requires exact cheap, frontier and per-web-arm final counts, directly and through the link" + + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +def test_projection_ignores_unregistered_paths(world, tmp_path, monkeypatch, kind): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, models = fresh_stage_design(w, web, tmp_path, "web" if kind == "linked" else kind) + root = Path(design["development_runs"]) + decoys = [ + root / cr.CHEAP[0] / "ablX/mctest/households/extra.json", + root / cr.FRONTIER[0] / "abl-unregistered/mctest/households/extra.json", + root / cr.CHEAP[0] / "pe3/unregistered-board/households/extra.json", + root / cr.CHEAP[0] / "pe3/mctest/households/not-json.txt", + root / cr.CHEAP[0] / "pe3/mctest/households/attempts/extra.json", + root / cr.WEB_MODELS[0] / "web/unregistered-board/households/extra.json", + root / cr.WEB_MODELS[0] / "web/mctest/households/not-json.txt", + root / cr.WEB_MODELS[0] / "web/mctest/households/attempts/extra.json", + root / cr.WEB_MODELS[0] / "unregistered-arm/mctest/households/extra.json", + ] + for path in decoys: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"totals": {"cost_usd": 1e12}})) + assert cr.preflight(design, models) == [], ( + "development projection ignores populated unregistered arms, boards, non-JSON files and nested attempts" + ) + From c46a0cdbaeca5501f1c4b27fff6678751caba80c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 13:43:35 -0500 Subject: [PATCH 20/31] Pin registered projection arithmetic with unequal populations and positive rounding ties. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 84 +++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index c15bed70..fb0695a9 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2692,7 +2692,7 @@ def test_resume_writes_the_operator_s_note(tmp_path): assert note["note"] == "key rotated" -def complete_web_design(world, tmp_path, monkeypatch, *, archives=False): +def complete_web_design(world, tmp_path, monkeypatch, *, archives=False, validate=True): """A web preflight with a genuinely ended synthetic calculator stage.""" from types import SimpleNamespace @@ -2753,7 +2753,8 @@ def git_pins(args, *a, **kw): monkeypatch.setenv( pilot.provider_key(cr.arm_profile(model, "web").model_id), "synthetic" ) - assert cr.preflight(design, cr.WEB_MODELS) == [] + if validate: + assert cr.preflight(design, cr.WEB_MODELS) == [] return w, design @@ -5435,6 +5436,7 @@ def test_command_line_resume_refuses_a_design_without_usable_runs( and capsys.readouterr().out.startswith("The run cannot start: ") ), "resume reports missing or unusable runs folders as startup refusals" + @pytest.mark.parametrize("kind", ["calculator", "web"]) def test_development_projection_refuses_a_fifo_without_blocking(tmp_path, kind): stage = cr.STAGES[kind] @@ -5478,6 +5480,7 @@ def read_projection(): and "FIFO" in str(outcome[0]) ), "development record reads refuse FIFO inputs promptly with a path-bearing error" + @pytest.mark.parametrize( "inventory,count,linked", [ @@ -5550,3 +5553,80 @@ def test_projection_ignores_unregistered_paths(world, tmp_path, monkeypatch, kin "development projection ignores populated unregistered arms, boards, non-JSON files and nested attempts" ) + +@pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) +def test_projection_preserves_registered_summation_and_pooling( + world, tmp_path, monkeypatch, kind +): + # A changed multiplier must reach this test's independent arithmetic oracle. + w, web = complete_web_design(world, tmp_path, monkeypatch, validate=False) + design, models = fresh_stage_design(w, web, tmp_path, "web" if kind == "linked" else kind) + link = Path(web["calculator"]["path"]) + target = json.loads(link.read_text()) if kind == "linked" else design + root = Path(target["development_runs"]) + + def replace_inventory(model, arms): + import shutil + + for existing in (root / model).iterdir(): + if (kind == "web" and existing.name in ("web", "web-pe3")) or ( + kind != "web" + and (existing.name in ("pe", "pe3") or existing.name.startswith("abl-")) + ): + shutil.rmtree(existing) + for arm, costs in arms: + # Both boards have nonuniform costs in each web arm. + for n, cost in enumerate(costs): + board = "mafree" if kind == "web" and n >= 50 else "mctest" + path = root / model / arm / board / "households" / f"{n:03}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"totals": {"cost_usd": cost}})) + + if kind == "web": + model = cr.WEB_MODELS[0] + costs = [1e16, 1.0] + [1e-16] * 98 + assert sum(costs) == 1e16 and math.fsum(costs) == 1e16 + 2 + replace_inventory( + model, + [("web", costs), ("web-pe3", list(range(1, 101)))], + ) + # Tiny positive costs move fsum across a rounding tie, including on + # Python 3.12, whose ordinary sum already compensates simple losses. + expected = {model: math.fsum([(1e16 + 2) / 100, 5050 / 100]) * 120} + else: + cheap, frontier = cr.CHEAP[0], cr.FRONTIER[0] + costs = [1e16, 1.0] + [1e-16] * 318 + assert sum(costs) == 1e16 and math.fsum(costs) == 1e16 + 2 + replace_inventory( + cheap, + [("abl-alpha", costs[:1]), ("abl-beta", costs[1:3]), ("pe3", costs[3:])], + ) + replace_inventory(frontier, [("pe", list(range(1, 11)))]) + # The registered calculator arithmetic uses ordinary sum and a pooled + # record mean despite unequal arm sizes of 1, 2 and 317 (320 total). + expected = {cheap: 1e16 / 320 * 120 * 7, frontier: 55 / 10 * 120 * 2} + # world uses six households to keep fake end-to-end stages small. Assert + # the real registered projection with its literal 120 households first. + stage = cr.STAGES["web" if kind == "web" else "calculator"] + fake_households = cr.HOUSEHOLDS + monkeypatch.setattr(cr, "HOUSEHOLDS", 120) + try: + got = cr.registered_expected_costs(root, stage) + finally: + monkeypatch.setattr(cr, "HOUSEHOLDS", fake_households) + assert {model: got[model] for model in expected} == expected, ( + "development projections preserve registered sum, fsum, pooled means and literal 120, 7 and 2 multipliers" + ) + target["expected_costs"].update( + {model: cost / 120 * fake_households for model, cost in expected.items()} + ) + total = math.fsum(target["expected_costs"].values()) + share = 120 if kind == "web" else 145 + target["caps"] = {m: share * c / total for m, c in target["expected_costs"].items()} + target["reserves"] = {m: 3 * c / cr.HOUSEHOLDS for m, c in target["expected_costs"].items()} + if kind == "linked": + link.write_text(json.dumps(target)) + design["calculator"]["sha256"] = cr.sha256_file(link) + assert cr.preflight(design, models) == [], ( + "development projections preserve registered sum, fsum, pooled means and literal 120, 7 and 2 multipliers" + ) From bdb3c536b2b97840c0975e779e58c5f0eaae3690 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 13:44:14 -0500 Subject: [PATCH 21/31] Assert the explicit object refusal so each command-line guard has a named reversion. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index fb0695a9..afd93d37 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5379,10 +5379,15 @@ def test_command_line_reports_unreadable_or_nonobject_designs( except Exception as error: raised = error result = None + output = capsys.readouterr().out assert ( raised is None and result == 1 - and capsys.readouterr().out.startswith("The run cannot start: ") + and output.startswith("The run cannot start: ") + and ( + fault not in ("null", "list") + or output.strip() == "The run cannot start: the design must be an object" + ) ), ( "the command line reports unreadable JSON and non-object designs as startup refusals" ) From 4e811bc24315cb7d9b9a2ff2e52628e50d124965 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 13:47:29 -0500 Subject: [PATCH 22/31] Wrap the new runner tests to satisfy the repository formatting checks. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 41 ++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index afd93d37..2be2b24e 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -3423,8 +3423,7 @@ def test_each_stage_allocation_accepts_exact_tolerance_and_refuses_next_float( if field == "caps": share = 120 if kind == "web" else 145 expected = { - m: (share - tolerance) / (len(stage.models) - 1) - for m in stage.models + m: (share - tolerance) / (len(stage.models) - 1) for m in stage.models } expected[model] = tolerance design = allocated_design(stage, expected) @@ -5360,7 +5359,8 @@ def test_preflight_refuses_negative_allocations_even_inside_formula_tolerances( path.write_text(json.dumps(target)) design["calculator"]["sha256"] = cr.sha256_file(path) assert cr.preflight(design, models) == want, ( - "allocations must be nonnegative even inside their formula tolerance; signed zero is allowed" + "allocations must be nonnegative even inside their formula tolerance; " + "signed zero is allowed" ) @@ -5389,7 +5389,8 @@ def test_command_line_reports_unreadable_or_nonobject_designs( or output.strip() == "The run cannot start: the design must be an object" ) ), ( - "the command line reports unreadable JSON and non-object designs as startup refusals" + "the command line reports unreadable JSON and non-object designs " + "as startup refusals" ) @@ -5505,7 +5506,11 @@ def read_projection(): for count in counts for linked in (False, True) ] - + [f"{inventory}-{count}-direct" for inventory in ("web", "web-pe3") for count in (99, 101)], + + [ + f"{inventory}-{count}-direct" + for inventory in ("web", "web-pe3") + for count in (99, 101) + ], ) def test_projection_refuses_missing_and_extra_finals_for_each_inventory( world, tmp_path, monkeypatch, inventory, count, linked @@ -5532,13 +5537,18 @@ def test_projection_refuses_missing_and_extra_finals_for_each_inventory( and f"expected {required}" in problems[0] and str(count) in problems[0] and (not linked or problems[0].startswith("linked calculator: ")) - ), "development projection requires exact cheap, frontier and per-web-arm final counts, directly and through the link" + ), ( + "development projection requires exact cheap, frontier and per-web-arm " + "final counts, directly and through the link" + ) @pytest.mark.parametrize("kind", ["calculator", "web", "linked"]) def test_projection_ignores_unregistered_paths(world, tmp_path, monkeypatch, kind): w, web = complete_web_design(world, tmp_path, monkeypatch) - design, models = fresh_stage_design(w, web, tmp_path, "web" if kind == "linked" else kind) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) root = Path(design["development_runs"]) decoys = [ root / cr.CHEAP[0] / "ablX/mctest/households/extra.json", @@ -5555,7 +5565,8 @@ def test_projection_ignores_unregistered_paths(world, tmp_path, monkeypatch, kin path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps({"totals": {"cost_usd": 1e12}})) assert cr.preflight(design, models) == [], ( - "development projection ignores populated unregistered arms, boards, non-JSON files and nested attempts" + "development projection ignores populated unregistered arms, boards, " + "non-JSON files and nested attempts" ) @@ -5565,7 +5576,9 @@ def test_projection_preserves_registered_summation_and_pooling( ): # A changed multiplier must reach this test's independent arithmetic oracle. w, web = complete_web_design(world, tmp_path, monkeypatch, validate=False) - design, models = fresh_stage_design(w, web, tmp_path, "web" if kind == "linked" else kind) + design, models = fresh_stage_design( + w, web, tmp_path, "web" if kind == "linked" else kind + ) link = Path(web["calculator"]["path"]) target = json.loads(link.read_text()) if kind == "linked" else design root = Path(target["development_runs"]) @@ -5620,7 +5633,8 @@ def replace_inventory(model, arms): finally: monkeypatch.setattr(cr, "HOUSEHOLDS", fake_households) assert {model: got[model] for model in expected} == expected, ( - "development projections preserve registered sum, fsum, pooled means and literal 120, 7 and 2 multipliers" + "development projections preserve registered sum, fsum, pooled means " + "and literal 120, 7 and 2 multipliers" ) target["expected_costs"].update( {model: cost / 120 * fake_households for model, cost in expected.items()} @@ -5628,10 +5642,13 @@ def replace_inventory(model, arms): total = math.fsum(target["expected_costs"].values()) share = 120 if kind == "web" else 145 target["caps"] = {m: share * c / total for m, c in target["expected_costs"].items()} - target["reserves"] = {m: 3 * c / cr.HOUSEHOLDS for m, c in target["expected_costs"].items()} + target["reserves"] = { + m: 3 * c / cr.HOUSEHOLDS for m, c in target["expected_costs"].items() + } if kind == "linked": link.write_text(json.dumps(target)) design["calculator"]["sha256"] = cr.sha256_file(link) assert cr.preflight(design, models) == [], ( - "development projections preserve registered sum, fsum, pooled means and literal 120, 7 and 2 multipliers" + "development projections preserve registered sum, fsum, pooled means " + "and literal 120, 7 and 2 multipliers" ) From 156356849b7c51034800aa1c29c9c915e42ba74f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 13:57:23 -0500 Subject: [PATCH 23/31] Keep lower allocation tolerance tests positive to isolate the formula check. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 2be2b24e..0533225f 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -3420,19 +3420,20 @@ def test_each_stage_allocation_accepts_exact_tolerance_and_refuses_next_float( stage = cr.STAGES[kind] model = stage.models[0] tolerance = 0.005 if field == "caps" else 0.0005 + allocation = 2 * tolerance if side < 0 else tolerance if field == "caps": share = 120 if kind == "web" else 145 expected = { - m: (share - tolerance) / (len(stage.models) - 1) for m in stage.models + m: (share - allocation) / (len(stage.models) - 1) for m in stage.models } - expected[model] = tolerance + expected[model] = allocation design = allocated_design(stage, expected) else: design = allocation_guard_design( - kind, target_cost=tolerance * cr.HOUSEHOLDS / 3 + kind, target_cost=allocation * cr.HOUSEHOLDS / 3 ) want = design[field][model] - assert want == tolerance + assert want == allocation boundary = want + side * tolerance assert abs(boundary - want) == tolerance design[field][model] = boundary From bc6fe74d23d8cb1ea2371ce07ae63ab9324ef0d6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 08:40:56 -0500 Subject: [PATCH 24/31] Check engine rereads after a successful preflight and retain all refusal messages. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_board_engine.py | 50 +++++++++++++++++++------ 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/tests/test_confirmatory_board_engine.py b/tests/test_confirmatory_board_engine.py index c7361703..d764ad9c 100644 --- a/tests/test_confirmatory_board_engine.py +++ b/tests/test_confirmatory_board_engine.py @@ -81,19 +81,23 @@ def test_engine_metadata_refuses_full_calculator_start( @pytest.mark.parametrize("fault", ["malformed", "unreadable"]) -def test_main_names_engine_reread_failure_and_preserves_all_problems( +def test_main_names_an_engine_reread_failure_after_a_clean_preflight( world, tmp_path, monkeypatch, capsys, fault ): w, design, metadata = engine_fault(world, tmp_path, monkeypatch, fault) - # Isolate main's final engine reread, which must retain preflight's problems. - monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + # Isolate main's final engine reread. Main reads the board only after a + # clean preflight, as test_confirmatory_run.py requires in + # test_command_line_prints_preflight_refusals_before_reading_the_board, + # so preflight here reports nothing. + monkeypatch.setattr(cr, "preflight", lambda *_: []) design_path = tmp_path / "design.json" design_path.write_text(json.dumps(design)) assert cr.main(["run", str(design_path)]) == 1 output = capsys.readouterr().out - assert "The run cannot start" in output - assert "earlier preflight problem" in output - assert str(metadata) in output + assert "The run cannot start:\n" in output, output + assert f"- {metadata}: the board's engine check failed: " in output, ( + "main's final engine reread names the board's metadata and its failure" + ) assert not w.provider.count @@ -126,17 +130,39 @@ def test_preflight_refuses_an_installed_engine_other_than_the_boards( assert not w.provider.count -def test_main_rechecks_the_installed_engine_and_preserves_all_problems( +def test_main_rechecks_the_installed_engine_after_a_clean_preflight( world, tmp_path, monkeypatch, capsys ): w, design = installed_elsewhere(world, tmp_path, monkeypatch) - # Isolate main's final engine check, which must retain preflight's problems. - monkeypatch.setattr(cr, "preflight", lambda *_: ["earlier preflight problem"]) + # Isolate main's final engine check, which runs only after a clean preflight. + monkeypatch.setattr(cr, "preflight", lambda *_: []) design_path = tmp_path / "design.json" design_path.write_text(json.dumps(design)) assert cr.main(["run", str(design_path)]) == 1 output = capsys.readouterr().out - assert "The run cannot start" in output - assert "earlier preflight problem" in output - assert "the board's engine check failed: policyengine-us 2.41.0" in output + assert "The run cannot start:\n" in output, output + assert "the board's engine check failed: policyengine-us 2.41.0" in output, ( + "main's final engine check refuses an installed engine other than the board's" + ) + assert not w.provider.count + + +@pytest.mark.parametrize("fault", ["malformed", "unreadable", "installed-elsewhere"]) +def test_main_prints_every_preflight_problem_and_starts_nothing( + world, tmp_path, monkeypatch, capsys, fault +): + if fault == "installed-elsewhere": + w, design = installed_elsewhere(world, tmp_path, monkeypatch) + else: + w, design, _ = engine_fault(world, tmp_path, monkeypatch, fault) + problems = ["earlier preflight problem", "a second preflight problem"] + monkeypatch.setattr(cr, "preflight", lambda *_: list(problems)) + design_path = tmp_path / "design.json" + design_path.write_text(json.dumps(design)) + assert cr.main(["run", str(design_path)]) == 1 + output = capsys.readouterr().out + assert "The run cannot start:\n" in output, output + assert all(f"- {problem}\n" in output for problem in problems), ( + "main prints every preflight problem" + ) assert not w.provider.count From 4032ab6dc111b2871bec4372e9e4507af6ccaa60 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 08:51:58 -0500 Subject: [PATCH 25/31] Keep every runner clock reading strictly increasing through wall-clock corrections. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 15 ++++-- tests/test_confirmatory_run.py | 92 +++++++++++++++++++++++++++++++--- 2 files changed, 94 insertions(+), 13 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index f0b174b8..07e988af 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -604,13 +604,18 @@ def attempt_cost(record: dict, *, saved: bool = True) -> float: def clock() -> float: - """`time.time()`, never earlier than a time it gave before. The wall clock - can step back by about 0.1 ms, even within a thread, and the run records order - attempts, stops, admission decisions and the ceiling's crossing by the - times this gives; each is taken under the lock that orders it.""" + """Return a time strictly greater than every earlier runner reading. + + When the wall clock repeats or steps back, advance by one representable + float. The lock orders attempts, stops, admission decisions and the + ceiling's crossing across all threads. + """ global _clock_last with _clock_lock: - _clock_last = max(time.time(), _clock_last) + now = time.time() + _clock_last = ( + now if now > _clock_last else math.nextafter(_clock_last, math.inf) + ) return _clock_last diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 0533225f..2e1fa729 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -46,10 +46,10 @@ TZ = "America/New_York" PREREG = "A frozen pre-registration for a test.\n" ENGINE = "2.60.4" -# The harness times requests with the raw wall clock, which steps back by -# about 0.1 ms on this host, even within a thread; the report allows a second, as -# here, which also covers a small correction of the clock. -CLOCK_SLACK = 1.0 +# The harness times requests with the raw wall clock. As in the report, allow +# 0.1 s plus the largest backward step measured from the host's sampled wall +# and monotonic clocks. +CLOCK_SLACK = 0.1 # The anchor command in a test: a copy of the anchors to the test's folder. ANCHOR_SCRIPT = Path(__file__).resolve().parent / "fake_anchor.py" # How far apart anchors and their triggers' stops may be in a test, on a @@ -745,13 +745,38 @@ def test_the_clock_never_goes_back(monkeypatch): monkeypatch.setattr(time, "time", lambda: next(readings)) assert [cr.clock() for _ in range(5)] == [ base, - base, - base + 2e-4, + math.nextafter(base, math.inf), base + 2e-4, + math.nextafter(base + 2e-4, math.inf), base + 3e-4, ] +def test_a_frozen_wall_clock_is_strictly_increasing_across_threads(monkeypatch): + base = time.time() + 1000.0 + monkeypatch.setattr(cr, "_clock_last", 0.0) + monkeypatch.setattr(time, "time", lambda: base) + readings = [[] for _ in range(8)] + ready = threading.Barrier(len(readings)) + + def read(values): + ready.wait() + values.extend(cr.clock() for _ in range(100)) + + threads = [threading.Thread(target=read, args=(values,)) for values in readings] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + values = [value for mine in readings for value in mine] + assert len(values) == 800 and len(set(values)) == len(values), ( + "runner clock values are strictly increasing across threads rule" + ) + assert all(all(a < b for a, b in zip(mine, mine[1:])) for mine in readings), ( + "runner clock values are strictly increasing across threads rule" + ) + + def test_a_backward_wall_clock_does_not_delay_a_due_anchor(monkeypatch): """Once the ordered clock reaches the due time, a wall-clock step cannot make the anchor not due again.""" @@ -787,6 +812,48 @@ def test_a_wall_clock_that_steps_back_keeps_the_run_procedure(world, monkeypatch assert len(w.attempts(model)) == N_DRAW * len(labels_of(model)) +@pytest.mark.parametrize("after", [0.1, 0.25]) +def test_a_two_second_wall_clock_step_keeps_the_run_procedure( + world, monkeypatch, after +): + monkeypatch.setattr(cr, "_clock_last", 0.0) + model = "gpt-6-sol" + w = world(models=(model,)) + sid = w.draw_order[0] + first_arm = cr.orders(w.run.master, model, w.draw_order, labels_of(model))[sid][0] + # Keep a real fake-provider request open through either drawn step point. + w.provider.specs[(model, first_arm, sid, 1)] = {"sleep": 0.35} + real_time, make_host = time.time, w.run.host_factory + started = [] + readings_after = [] + + def start_host(folder): + host = make_host(folder) + # Time the step from a sampled run, so a slow startup cannot hide it. + started.append(time.monotonic()) + return host + + def stepped_time(): + now = real_time() + if started and time.monotonic() - started[0] >= after: + readings_after.append(now) + return now - 2.0 + return now + + w.run.host_factory = start_host + monkeypatch.setattr(time, "time", stepped_time) + w.execute() + assert readings_after, "the two-second wall-clock step must occur during the run" + assert host_from_files(w.runs() / "host").back > 1.9 + problems = run_problems(w) + assert problems == [], ( + f"runner procedure survives a two-second wall-clock step rule: {problems}" + ) + assert len(w.attempts(model)) == N_DRAW * len(labels_of(model)), ( + "runner procedure survives a two-second wall-clock step rule" + ) + + def test_the_ceiling_crosses_at_its_amount_and_stays_crossed(): ceiling = cr.Ceiling(1.0, 0.25) ceiling.add(0.5) @@ -1066,6 +1133,11 @@ def host_from_files(folder: Path): samples = [ json.loads(x) for x in (folder / "host-log.jsonl").read_text().splitlines() if x ] + back = 0.0 + for first, second in zip(samples, samples[1:]): + elapsed = second["monotonic"] - first["monotonic"] + if elapsed >= 0: + back = max(back, elapsed - (second["time"] - first["time"])) spans = cr.guard_spans( (folder / "clamshell-guard.log").read_text(), meta["timezone"], end ) @@ -1079,6 +1151,9 @@ def host_from_files(folder: Path): pressure = [float(s["time"]) for s in good if int(s["pressure"]) > 1] class Host: + def __init__(self): + self.back = back + def trouble(self, s, e): for a, b, what in spans: if a <= e and b >= s: @@ -1159,6 +1234,7 @@ def run_problems(world: World) -> list[str]: """Every rule of the run procedure, recomputed from the runner's files.""" problems: list[str] = [] host = host_from_files(world.runs() / "host") + clock_slack = CLOCK_SLACK + host.back frozen = world.design["frozen_sha256"] problems += chain_problems(world.runs() / "host" / "host-log.jsonl", cr.HOST_SEED) problems += anchor_problems(world) @@ -1255,8 +1331,8 @@ def run_problems(world: World) -> list[str]: for key in ("request_started_at", "request_completed_at") ] if times and ( - min(times) < x["start"] - CLOCK_SLACK - or max(times) > x["end"] + CLOCK_SLACK + min(times) < x["start"] - clock_slack + or max(times) > x["end"] + clock_slack ): problems.append( f"{at}#{k}: requests {min(times)!r}-{max(times)!r} outside " From bb381443695a7983303829ba9f5cabff15c388f7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 11:50:44 -0500 Subject: [PATCH 26/31] Bind anchor clones to the registered stage branch and remote before spending. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 65 ++++++++ tests/fake_anchor.py | 12 +- tests/test_confirmatory_run.py | 261 ++++++++++++++++++++++++++++++++- 3 files changed, 333 insertions(+), 5 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 07e988af..94934bb5 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -145,6 +145,11 @@ import tool_pilot_argon as pilot # noqa: E402 TREATMENT_COMMIT = "c0a2885bfc8f5df59a93458230c445befb986dcb" +ANCHOR_REPOSITORY = "PolicyEngine/policybench-v2-anchors" +ANCHOR_REFS = { + "calculator": "refs/heads/anchors/v2-confirmatory", + "web": "refs/heads/anchors/v2-confirmatory-web", +} # The treatment's code, which must be the treatment commit's. TREATMENT_PATHS = ( "scripts/tool_pilot_argon.py", @@ -2921,6 +2926,65 @@ def projection_problems(design: dict, stage: Stage) -> list[str]: return [f"the registered expected-cost projection: {error}"] +def anchor_clone_problems(command, stage: Stage) -> list[str]: + """Check only local anchor-clone metadata; never fetch or push.""" + arguments = [str(value) for value in command] if isinstance(command, list) else [] + repositories = [] + for index, argument in enumerate(arguments): + if argument == "--repo": + repositories.append( + arguments[index + 1] if index + 1 < len(arguments) else "" + ) + elif argument.startswith("--repo="): + repositories.append(argument.removeprefix("--repo=")) + if ( + len(repositories) != 1 + or not repositories[0] + or repositories[0].startswith("--") + ): + return ["the anchor command must name exactly one --repo clone"] + repo = Path(repositories[0]).expanduser() + + def git(*args): + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True + ) + + try: + tree = git("rev-parse", "--is-inside-work-tree") + head = git("rev-parse", "--verify", "HEAD^{commit}") + if tree.returncode or tree.stdout.strip() != "true" or head.returncode: + return [ + f"the anchor clone {repo}: needs a working tree " + "with a checked-out commit" + ] + problems = [] + branch = git("symbolic-ref", "--quiet", "HEAD") + expected = ANCHOR_REFS[stage.kind] + if branch.returncode or branch.stdout.strip() != expected: + problems.append( + f"the anchor clone {repo}: checked-out branch must be {expected}" + ) + origin = git("config", "--get", "remote.origin.url") + accepted = { + f"{prefix}{ANCHOR_REPOSITORY}{suffix}" + for prefix in ( + "https://github.com/", + "git@github.com:", + "ssh://git@github.com/", + ) + for suffix in ("", ".git") + } + if origin.returncode or origin.stdout.strip() not in accepted: + problems.append( + f"the anchor clone {repo}: origin must name {ANCHOR_REPOSITORY} " + "over SSH or HTTPS" + ) + return problems + except OSError as error: + return [f"the anchor clone {repo}: cannot read Git metadata: {error}"] + + def preflight(design: dict, models: tuple[str, ...]) -> list[str]: """Return recorded input problems for every malformed design.""" try: @@ -3001,6 +3065,7 @@ def _preflight(design: dict, models: tuple[str, ...]) -> list[str]: or shutil.which(str(command[0])) is None ): problems.append(f"the anchor command {command!r} is not a command here") + problems.extend(anchor_clone_problems(command, stage)) problems.extend(rehearsal_problems(design)) preregistration = Path(design["preregistration"]) try: diff --git a/tests/fake_anchor.py b/tests/fake_anchor.py index 73b5c32f..5ee668ca 100644 --- a/tests/fake_anchor.py +++ b/tests/fake_anchor.py @@ -1,13 +1,17 @@ """The confirmatory runner's anchor command in its tests: a copy of the run's anchors.jsonl to FOLDER/anchored.jsonl. -Usage: python fake_anchor.py FOLDER ANCHORS +Usage: python fake_anchor.py [--repo CLONE] FOLDER ANCHORS """ +import argparse import shutil -import sys from pathlib import Path -folder, anchors = Path(sys.argv[1]), Path(sys.argv[2]) -shutil.copyfile(anchors, folder / "anchored.jsonl") +parser = argparse.ArgumentParser() +parser.add_argument("--repo", type=Path) +parser.add_argument("folder", type=Path) +parser.add_argument("anchors", type=Path) +args = parser.parse_args() +shutil.copyfile(args.anchors, args.folder / "anchored.jsonl") print("anchored") diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 2e1fa729..bdf9e97c 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -2561,6 +2561,71 @@ def synthetic_development_tree(root, kind, expected): ) +def synthetic_anchor_clone(tmp_path, kind): + """A local clone with synthetic history and registered remote metadata.""" + branches = { + "calculator": "anchors/v2-confirmatory", + "web": "anchors/v2-confirmatory-web", + } + source = tmp_path.parent / "anchor-fixture-source" + if not source.exists(): + subprocess.run( + [ + "git", + "init", + "--quiet", + "--template=", + "--initial-branch=fixture", + str(source), + ], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(source), + "-c", + "user.name=Anchor fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "core.hooksPath=/dev/null", + "-c", + "commit.gpgsign=false", + "commit", + "--quiet", + "--allow-empty", + "-m", + "Create synthetic anchor history.", + ], + check=True, + ) + repo = tmp_path / f"anchor-clone-{kind}" + if not repo.exists(): + subprocess.run( + ["git", "clone", "--quiet", "--shared", str(source), str(repo)], + check=True, + ) + subprocess.run( + ["git", "-C", str(repo), "checkout", "--quiet", "-b", branches[kind]], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(repo), + "remote", + "set-url", + "origin", + "https://github.com/PolicyEngine/policybench-v2-anchors.git", + ], + check=True, + ) + return repo + + def complete_preflight_design(w, tmp_path): """Each calculator preflight starts after standing no-tools finals that pass the no-tools script's treatment checks: built through the harness from the @@ -2608,6 +2673,13 @@ def complete_preflight_design(w, tmp_path): "stage": "confirm", }, development_runs=str(tmp_path / "development-runs"), + anchor_command=[ + sys.executable, + str(ANCHOR_SCRIPT), + "--repo", + str(synthetic_anchor_clone(tmp_path, "calculator")), + str(tmp_path), + ], ) synthetic_development_tree( Path(design["development_runs"]), "calculator", design["expected_costs"] @@ -2806,6 +2878,13 @@ def complete_web_design(world, tmp_path, monkeypatch, *, archives=False, validat spent_before=math.fsum( [w.design["spent_before"], *cr.calculator_evidence(w.design)] ), + anchor_command=[ + sys.executable, + str(ANCHOR_SCRIPT), + "--repo", + str(synthetic_anchor_clone(tmp_path, "web")), + str(tmp_path), + ], ) del design["frontier_arms"] synthetic_development_tree( @@ -2817,7 +2896,7 @@ def complete_web_design(world, tmp_path, monkeypatch, *, archives=False, validat real = subprocess.run def git_pins(args, *a, **kw): - if args and args[0] == "git": + if list(args) == ["git", "-C", str(cr.ROOT), "rev-parse", "HEAD"]: return SimpleNamespace(returncode=0, stdout=cr.TREATMENT_COMMIT + "\n") return real(args, *a, **kw) @@ -5729,3 +5808,183 @@ def replace_inventory(model, arms): "development projections preserve registered sum, fsum, pooled means " "and literal 120, 7 and 2 multipliers" ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize( + "origin", + [ + prefix + "PolicyEngine/policybench-v2-anchors" + suffix + for prefix in ( + "https://github.com/", + "git@github.com:", + "ssh://git@github.com/", + ) + for suffix in ("", ".git") + ], +) +def test_anchor_clone_accepts_registered_branch_and_origin(tmp_path, kind, origin): + repo = synthetic_anchor_clone(tmp_path, kind) + subprocess.run( + ["git", "-C", str(repo), "remote", "set-url", "origin", origin], check=True + ) + command = [sys.executable, "anchor.py", "--repo", str(repo)] + assert cr.anchor_clone_problems(command, cr.STAGES[kind]) == [], ( + "anchor clone registered branch and SSH or HTTPS origin rule" + ) + command[-2:] = [f"--repo={repo}"] + assert cr.anchor_clone_problems(command, cr.STAGES[kind]) == [], ( + "anchor clone registered branch and SSH or HTTPS origin rule" + ) + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize( + "fault", + [ + "branch", + "detached", + "origin", + "no-origin", + "no-repo", + "two-repos", + "missing-value", + ], +) +def test_anchor_clone_refuses_unregistered_metadata(tmp_path, kind, fault): + repo = synthetic_anchor_clone(tmp_path, kind) + command = [sys.executable, "anchor.py", "--repo", str(repo)] + expected_ref = { + "calculator": "refs/heads/anchors/v2-confirmatory", + "web": "refs/heads/anchors/v2-confirmatory-web", + }[kind] + if fault == "branch": + subprocess.run( + ["git", "-C", str(repo), "checkout", "--quiet", "-b", "unregistered"], + check=True, + ) + elif fault == "detached": + subprocess.run( + ["git", "-C", str(repo), "checkout", "--quiet", "--detach"], check=True + ) + elif fault == "origin": + subprocess.run( + [ + "git", + "-C", + str(repo), + "remote", + "set-url", + "origin", + "https://github.com/example/unregistered.git", + ], + check=True, + ) + elif fault == "no-origin": + subprocess.run( + ["git", "-C", str(repo), "remote", "remove", "origin"], check=True + ) + elif fault == "no-repo": + command = command[:2] + elif fault == "two-repos": + command += [f"--repo={repo}"] + else: + command.pop() + if fault in ("no-repo", "two-repos", "missing-value"): + expected = "the anchor command must name exactly one --repo clone" + rule = "anchor command names exactly one repo clone rule" + elif fault in ("branch", "detached"): + expected = f"the anchor clone {repo}: checked-out branch must be {expected_ref}" + rule = "anchor clone registered checked-out stage branch rule" + else: + expected = ( + f"the anchor clone {repo}: origin must name " + "PolicyEngine/policybench-v2-anchors over SSH or HTTPS" + ) + rule = "anchor clone registered origin rule" + raised, problems = None, None + try: + problems = cr.anchor_clone_problems(command, cr.STAGES[kind]) + except (IndexError, OSError) as error: + raised = error + assert raised is None and problems == [expected], rule + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +@pytest.mark.parametrize("fault", ["bare", "unborn"]) +def test_anchor_clone_requires_a_working_tree_and_commit(tmp_path, kind, fault): + repo = tmp_path / "anchor" + branch = { + "calculator": "anchors/v2-confirmatory", + "web": "anchors/v2-confirmatory-web", + }[kind] + if fault == "bare": + source = synthetic_anchor_clone(tmp_path, kind) + subprocess.run( + ["git", "clone", "--quiet", "--bare", str(source), str(repo)], check=True + ) + else: + subprocess.run( + [ + "git", + "init", + "--quiet", + "--template=", + f"--initial-branch={branch}", + str(repo), + ], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(repo), + "config", + "remote.origin.url", + "https://github.com/PolicyEngine/policybench-v2-anchors.git", + ], + check=True, + ) + assert cr.anchor_clone_problems( + ["anchor", "--repo", str(repo)], cr.STAGES[kind] + ) == [f"the anchor clone {repo}: needs a working tree with a checked-out commit"], ( + "anchor clone requires a working tree and checked-out commit rule" + ) + + +def test_anchor_clone_git_read_failure_is_a_refusal(tmp_path, monkeypatch): + repo = tmp_path / "clone" + + def denied(*args, **kwargs): + raise PermissionError("synthetic anchor metadata denial") + + monkeypatch.setattr(cr.subprocess, "run", denied) + raised, problems = None, None + try: + problems = cr.anchor_clone_problems( + ["anchor", "--repo", str(repo)], cr.STAGES["web"] + ) + except OSError as error: + raised = error + assert raised is None and problems == [ + f"the anchor clone {repo}: cannot read Git metadata: " + "synthetic anchor metadata denial" + ], "anchor clone Git read errors become recorded refusals rule" + + +def test_preflight_checks_registered_anchor_clone(world, tmp_path, monkeypatch): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + sentinel = "synthetic registered anchor clone refusal" + seen = [] + + def refused(command, stage): + seen.append((command, stage.kind)) + return [sentinel] + + monkeypatch.setattr(cr, "anchor_clone_problems", refused) + problems = cr.preflight(design, (cr.CHEAP[0],)) + assert ( + seen == [(design["anchor_command"], "calculator")] and sentinel in problems + ), "preflight applies registered anchor clone checks rule" From 616a9ebf2572fada743de4ae2917218625f04aa4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 12:03:34 -0500 Subject: [PATCH 27/31] Bind run manifests and restart checks to the exact design file bytes. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 62 +++++++++++++++++++--- tests/test_confirmatory_run.py | 96 +++++++++++++++++++++++++++++++++- 2 files changed, 150 insertions(+), 8 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 94934bb5..19a218b2 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -1688,6 +1688,7 @@ def timeout(profile): board_path = Path(design["board"]) return { + "design_sha256": self.run.design_sha256, "started_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "git_head": pilot._git("rev-parse", "HEAD"), "git_dirty": bool(pilot._git("status", "--porcelain")), @@ -1824,8 +1825,11 @@ def __init__( completion_for: Callable = default_completion, host_factory: Callable | None = None, out=sys.stdout, + *, + design_sha256: str, ): self.design, self.out = design, out + self.design_sha256 = design_sha256 self.stage = stage_for(design) self.workers = int(design["workers"]) if workers is None else workers self.converse, self.pool_factory = converse, pool_factory @@ -2985,12 +2989,52 @@ def git(*args): return [f"the anchor clone {repo}: cannot read Git metadata: {error}"] -def preflight(design: dict, models: tuple[str, ...]) -> list[str]: +def restart_manifest_problems( + design: dict, stage: Stage, design_sha256: str | None +) -> list[str]: + """Bind every existing registered-model manifest to the input file's bytes.""" + from confirmatory_no_tools import read_regular, saved_children + + problems = [] + valid_hash = isinstance(design_sha256, str) and re.fullmatch( + r"[0-9a-f]{64}", design_sha256 + ) + for model in stage.models: + folder = Path(design["runs"]) / model + try: + paths = [ + path + for path in saved_children(folder) + if path.name.startswith("manifest-") and path.suffix == ".json" + ] + except ValueError as error: + problems.append(str(error)) + continue + for path in paths: + try: + manifest = json.loads(read_regular(path).decode("utf-8")) + except (OSError, ValueError, RecursionError) as error: + problems.append(f"{path}: cannot read restart manifest: {error}") + continue + if ( + not valid_hash + or not isinstance(manifest, dict) + or manifest.get("design_sha256") != design_sha256 + ): + problems.append( + f"{path}: design_sha256 is missing or differs from the design file" + ) + return problems + + +def preflight( + design: dict, models: tuple[str, ...], design_sha256: str | None = None +) -> list[str]: """Return recorded input problems for every malformed design.""" try: if not isinstance(design, dict): raise ValueError("the design must be an object") - return _preflight(design, models) + return _preflight(design, models, design_sha256) except ( OSError, ValueError, @@ -3004,7 +3048,9 @@ def preflight(design: dict, models: tuple[str, ...]) -> list[str]: return [f"the design is malformed: {error}"] -def _preflight(design: dict, models: tuple[str, ...]) -> list[str]: +def _preflight( + design: dict, models: tuple[str, ...], design_sha256: str | None = None +) -> list[str]: """What stops the run from starting.""" missing = [k for k in REQUIRED if k not in design] if missing: @@ -3015,7 +3061,7 @@ def _preflight(design: dict, models: tuple[str, ...]) -> list[str]: return [str(error)] if stage.kind == "calculator" and "frontier_arms" not in design: return ["the design has no frontier_arms"] - problems = [] + problems = restart_manifest_problems(design, stage, design_sha256) if ( stage.kind == "calculator" and design.get("overall_cap") != CALCULATOR_OVERALL_CAP @@ -3217,7 +3263,9 @@ def main(argv=None) -> int: resume.add_argument("--note", default="the account was fixed") args = parser.parse_args(argv) try: - design = json.loads(args.design.read_text()) + design_bytes = args.design.read_bytes() + design = json.loads(design_bytes.decode("utf-8")) + design_sha256 = sha256_bytes(design_bytes) if not isinstance(design, dict): raise ValueError("the design must be an object") stage = stage_for(design) @@ -3243,7 +3291,7 @@ def main(argv=None) -> int: ) return 1 models = stage.models - problems = preflight(design, models) + problems = preflight(design, models, design_sha256) if problems: print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) return 1 @@ -3260,7 +3308,7 @@ def main(argv=None) -> int: if problems: print("The run cannot start:\n" + "\n".join(f"- {p}" for p in problems)) return 1 - plan = Run(design) + plan = Run(design, design_sha256=design_sha256) if args.dry_run: for model, lanes in plan.lanes.items(): sizes = [len(lanes.households(lane)) for lane in (1, 2)] diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index bdf9e97c..4ad552e0 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -173,7 +173,7 @@ def test_design_kind_is_required_and_registered_before_any_stage_starts(kind): with pytest.raises(ValueError, match="kind must be calculator or web"): cr.stage_for(design) with pytest.raises(ValueError, match="kind must be calculator or web"): - cr.Run(design) + cr.Run(design, design_sha256="0" * 64) @pytest.mark.parametrize("label", cr.WEB_ARMS) @@ -1084,6 +1084,7 @@ def __init__( self.provider = FakeProvider(self, specs) self.run = cr.Run( self.design, + design_sha256=cr.sha256_bytes(json.dumps(self.design).encode("utf-8")), models=self.models, converse=self.provider, pool_factory=pool, @@ -1093,6 +1094,10 @@ def __init__( ) def execute(self): + self.design_path = self.tmp / "design.json" + data = json.dumps(self.design).encode("utf-8") + self.design_path.write_bytes(data) + self.run.design_sha256 = cr.sha256_bytes(data) self.run.execute() return self @@ -1538,6 +1543,9 @@ def test_an_untouched_run_runs_every_household_and_arm(world): manifests = list((w.runs() / model).glob("manifest-*.json")) assert len(manifests) == 1 m = json.loads(manifests[0].read_text()) + assert m["design_sha256"] == cr.sha256_file(w.design_path), ( + "every manifest binds the exact design file bytes rule" + ) assert m["arms"] == { label: { "arm": cr.arm_string(label), @@ -5988,3 +5996,89 @@ def refused(command, stage): assert ( seen == [(design["anchor_command"], "calculator")] and sentinel in problems ), "preflight applies registered anchor clone checks rule" + + +@pytest.mark.parametrize("style", ["lf", "crlf", "spaces"]) +def test_run_manifests_hash_the_design_files_exact_bytes( + world, tmp_path, monkeypatch, style +): + """Exercise the real CLI reader and manifest writer with a fake provider.""" + w = world(models=(cr.FRONTIER[0],)) + text = json.dumps(w.design, indent=2) + if style == "crlf": + text = text.replace("\n", "\r\n") + elif style == "spaces": + text = " \n" + text + "\n " + data = text.encode("utf-8") + path = tmp_path / "cli-design.json" + path.write_bytes(data) + reads = [] + real_read = Path.read_bytes + + def counted_read(source): + if source == path: + reads.append(source) + return real_read(source) + + def plan(design, *, design_sha256): + assert design == w.design + w.run.design_sha256 = design_sha256 + return w.run + + monkeypatch.setattr(Path, "read_bytes", counted_read) + monkeypatch.setattr(cr, "preflight", lambda *args: []) + monkeypatch.setattr(pilot, "_require_engine", lambda _: None) + monkeypatch.setattr(cr, "Run", plan) + assert cr.main(["run", str(path)]) == 0 + manifests = list(w.runs().glob("*/manifest-*.json")) + expected = hashlib.sha256(data).hexdigest() + assert manifests and all( + json.loads(manifest.read_text())["design_sha256"] == expected + for manifest in manifests + ), "every manifest binds the exact design file bytes rule" + assert reads == [path], "the runner reads the design file bytes once rule" + + +@pytest.mark.parametrize("kind", ["calculator", "web"]) +def test_preflight_binds_every_registered_restart_manifest( + world, tmp_path, monkeypatch, kind +): + w, web = complete_web_design(world, tmp_path, monkeypatch) + design, _ = fresh_stage_design(w, web, tmp_path, kind) + stage = cr.STAGES[kind] + models = (stage.models[0],) + data = json.dumps(design).encode("utf-8") + path = tmp_path / "restart-design.json" + path.write_bytes(data) + expected = hashlib.sha256(data).hexdigest() + assert cr.preflight(design, models, expected) == [], "bound restart control rule" + # Inspect every registered folder even when the requested models are a subset. + manifest = Path(design["runs"]) / stage.models[-1] / "manifest-prior.json" + manifest.parent.mkdir(parents=True) + manifest.write_text(json.dumps({"design_sha256": expected})) + decoy_model = cr.FRONTIER[0] if kind == "web" else "claude-sonnet-5.5" + decoy = Path(design["runs"]) / decoy_model / "manifest-decoy.json" + decoy.parent.mkdir(parents=True) + decoy.write_text('{"design_sha256": "unregistered"}') + assert cr.preflight(design, models, expected) == [], "bound restart control rule" + refusal = f"{manifest}: design_sha256 is missing or differs from the design file" + for value in ({}, {"design_sha256": "0" * 64}, []): + manifest.write_text(json.dumps(value)) + assert cr.preflight(design, models, expected) == [refusal], ( + "restart manifests bind every registered model " + "to the same design bytes rule" + ) + manifest.write_text(json.dumps({"design_sha256": expected})) + path.write_bytes(data + b" \n") + changed = hashlib.sha256(path.read_bytes()).hexdigest() + assert cr.preflight(design, models, changed) == [refusal], ( + "restart manifests bind every registered model to the same design bytes rule" + ) + assert cr.preflight(design, models) == [refusal], ( + "restart manifests require an explicit expected design digest rule" + ) + manifest.write_text("{") + problems = cr.preflight(design, models, expected) + assert len(problems) == 1 and problems[0].startswith( + f"{manifest}: cannot read restart manifest: " + ), "malformed restart manifests become recorded refusals rule" From d5ec8d04113822b2365e774143cd758dbb26621c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 13:00:51 -0500 Subject: [PATCH 28/31] Refuse invalid resume models and retain every preflight problem. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 25 +++++++++--- tests/test_confirmatory_run.py | 71 ++++++++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 19a218b2..c20085b8 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -3031,10 +3031,11 @@ def preflight( design: dict, models: tuple[str, ...], design_sha256: str | None = None ) -> list[str]: """Return recorded input problems for every malformed design.""" + problems = [] try: if not isinstance(design, dict): raise ValueError("the design must be an object") - return _preflight(design, models, design_sha256) + return _preflight(design, models, design_sha256, problems) except ( OSError, ValueError, @@ -3045,11 +3046,14 @@ def preflight( OverflowError, RuntimeError, ) as error: - return [f"the design is malformed: {error}"] + return [*problems, f"the design is malformed: {error}"] def _preflight( - design: dict, models: tuple[str, ...], design_sha256: str | None = None + design: dict, + models: tuple[str, ...], + design_sha256: str | None = None, + problems: list[str] | None = None, ) -> list[str]: """What stops the run from starting.""" missing = [k for k in REQUIRED if k not in design] @@ -3061,7 +3065,9 @@ def _preflight( return [str(error)] if stage.kind == "calculator" and "frontier_arms" not in design: return ["the design has no frontier_arms"] - problems = restart_manifest_problems(design, stage, design_sha256) + if problems is None: + problems = [] + problems.extend(restart_manifest_problems(design, stage, design_sha256)) if ( stage.kind == "calculator" and design.get("overall_cap") != CALCULATOR_OVERALL_CAP @@ -3262,6 +3268,9 @@ def main(argv=None) -> int: resume.add_argument("model") resume.add_argument("--note", default="the account was fixed") args = parser.parse_args(argv) + refusal_prefix = ( + "No resume requested" if args.command == "resume" else "The run cannot start" + ) try: design_bytes = args.design.read_bytes() design = json.loads(design_bytes.decode("utf-8")) @@ -3270,17 +3279,21 @@ def main(argv=None) -> int: raise ValueError("the design must be an object") stage = stage_for(design) except (OSError, ValueError, TypeError, AttributeError, RecursionError) as error: - print(f"The run cannot start: {error}") + print(f"{refusal_prefix}: {error}") return 1 if args.command == "resume": try: + if args.model not in stage.models: + raise ValueError( + f"model {args.model!r} is not in the {stage.kind} stage" + ) runs = design.get("runs") if not isinstance(runs, str) or not runs.strip(): raise ValueError("the design must name a usable runs folder") folder = Path(runs) / args.model write_json(folder / "RESUME.json", {"time": time.time(), "note": args.note}) except (OSError, ValueError, TypeError) as error: - print(f"The run cannot start: {error}") + print(f"No resume requested: {error}") return 1 print(f"{args.model}: resume requested; the runner resumes within a second") return 0 diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 4ad552e0..790eca57 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5544,13 +5544,14 @@ def test_command_line_reports_unreadable_or_nonobject_designs( raised = error result = None output = capsys.readouterr().out + prefix = "No resume requested" if command == "resume" else "The run cannot start" assert ( raised is None and result == 1 - and output.startswith("The run cannot start: ") + and output.startswith(f"{prefix}: ") and ( fault not in ("null", "list") - or output.strip() == "The run cannot start: the design must be an object" + or output.strip() == f"{prefix}: the design must be an object" ) ), ( "the command line reports unreadable JSON and non-object designs " @@ -5603,7 +5604,7 @@ def test_command_line_resume_refuses_a_design_without_usable_runs( assert ( raised is None and result == 1 - and capsys.readouterr().out.startswith("The run cannot start: ") + and capsys.readouterr().out.startswith("No resume requested: ") ), "resume reports missing or unusable runs folders as startup refusals" @@ -6082,3 +6083,67 @@ def test_preflight_binds_every_registered_restart_manifest( assert len(problems) == 1 and problems[0].startswith( f"{manifest}: cannot read restart manifest: " ), "malformed restart manifests become recorded refusals rule" + + +@pytest.mark.parametrize( + "kind,model", + [ + ("calculator", "gpt-6-luuna"), + ("calculator", "../x"), + ("calculator", "claude-sonnet-5.5"), + ("web", "gpt-6-sol"), + ], +) +def test_resume_refuses_models_outside_its_registered_stage( + tmp_path, capsys, kind, model +): + design = tmp_path / "design.json" + design.write_text(json.dumps({"kind": kind, "runs": str(tmp_path / "runs")})) + before = set(tmp_path.rglob("*")) + result = cr.main(["resume", str(design), model]) + assert result == 1 and capsys.readouterr().out == ( + f"No resume requested: model {model!r} is not in the {kind} stage\n" + ), "resume accepts only registered stage models before writing rule" + assert set(tmp_path.rglob("*")) == before, ( + "resume accepts only registered stage models before writing rule" + ) + + +def test_resume_reports_write_failures_as_no_resume_requested( + tmp_path, capsys, monkeypatch +): + design = tmp_path / "design.json" + design.write_text( + json.dumps({"kind": "calculator", "runs": str(tmp_path / "runs")}) + ) + + def denied(*args, **kwargs): + raise PermissionError("synthetic resume write denial") + + monkeypatch.setattr(cr, "write_json", denied) + assert cr.main(["resume", str(design), "gpt-6-luna"]) == 1 + assert ( + capsys.readouterr().out + == "No resume requested: synthetic resume write denial\n" + ), "every resume write refusal uses the no-resume prefix rule" + assert not (tmp_path / "runs").exists() + + +def test_preflight_catch_all_keeps_earlier_problems( + world, tmp_path, monkeypatch, capsys +): + w = world(models=(cr.CHEAP[0],)) + design = complete_preflight_design(w, tmp_path) + design.update(overall_cap=149.0, board=None) + path = tmp_path / "malformed-design.json" + path.write_text(json.dumps(design)) + monkeypatch.setattr( + pilot, "_require_engine", lambda _: pytest.fail("preflight must refuse first") + ) + assert cr.main(["run", str(path)]) == 1 + output = capsys.readouterr().out + early = "- the calculator design's overall_cap must be 150\n" + late = "- the design is malformed: " + assert ( + early in output and late in output and output.index(early) < output.index(late) + ), "preflight catch-all retains earlier problems before malformed design rule" From 8f6a74284bbf69d0b14e7223843cbf6ee9ab42d0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 13:06:45 -0500 Subject: [PATCH 29/31] Assert complete command-line refusals and cover missing stage kinds. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 40 ++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 790eca57..44f573ec 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -5529,13 +5529,23 @@ def test_preflight_refuses_negative_allocations_even_inside_formula_tolerances( @pytest.mark.parametrize("command", ["run", "resume"]) -@pytest.mark.parametrize("fault", ["missing", "invalid-json", "null", "list"]) +@pytest.mark.parametrize( + "fault", ["missing", "invalid-json", "null", "list", "no-kind", "bad-kind"] +) def test_command_line_reports_unreadable_or_nonobject_designs( tmp_path, capsys, command, fault ): path = tmp_path / "design.json" if fault != "missing": - path.write_text({"invalid-json": "{", "null": "null", "list": "[]"}[fault]) + path.write_text( + { + "invalid-json": "{", + "null": "null", + "list": "[]", + "no-kind": "{}", + "bad-kind": '{"kind": "unregistered"}', + }[fault] + ) argv = [command, str(path)] + (["gpt-6-luna"] if command == "resume" else []) raised = None try: @@ -5545,15 +5555,22 @@ def test_command_line_reports_unreadable_or_nonobject_designs( result = None output = capsys.readouterr().out prefix = "No resume requested" if command == "resume" else "The run cannot start" - assert ( - raised is None - and result == 1 - and output.startswith(f"{prefix}: ") - and ( - fault not in ("null", "list") - or output.strip() == f"{prefix}: the design must be an object" + reason = { + "missing": f"[Errno 2] No such file or directory: {str(path)!r}", + "invalid-json": ( + "Expecting property name enclosed in double quotes: " + "line 1 column 2 (char 1)" + ), + "null": "the design must be an object", + "list": "the design must be an object", + "no-kind": "the design's kind must be calculator or web", + "bad-kind": "the design's kind must be calculator or web", + }[fault] + if fault in ("no-kind", "bad-kind"): + assert raised is None and result == 1 and output == f"{prefix}: {reason}\n", ( + "command line handles stage selection inside the startup guard rule" ) - ), ( + assert raised is None and result == 1 and output == f"{prefix}: {reason}\n", ( "the command line reports unreadable JSON and non-object designs " "as startup refusals" ) @@ -5604,7 +5621,8 @@ def test_command_line_resume_refuses_a_design_without_usable_runs( assert ( raised is None and result == 1 - and capsys.readouterr().out.startswith("No resume requested: ") + and capsys.readouterr().out + == "No resume requested: the design must name a usable runs folder\n" ), "resume reports missing or unusable runs folders as startup refusals" From ca60e565f8a404e1a59d73198d754024846d17c8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 13:54:39 -0500 Subject: [PATCH 30/31] Keep the clock-step admission counterexample observable under host pauses. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 52 ++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 44f573ec..6d635939 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -819,28 +819,62 @@ def test_a_two_second_wall_clock_step_keeps_the_run_procedure( monkeypatch.setattr(cr, "_clock_last", 0.0) model = "gpt-6-sol" w = world(models=(model,)) - sid = w.draw_order[0] - first_arm = cr.orders(w.run.master, model, w.draw_order, labels_of(model))[sid][0] - # Keep a real fake-provider request open through either drawn step point. - w.provider.specs[(model, first_arm, sid, 1)] = {"sleep": 0.35} - real_time, make_host = time.time, w.run.host_factory + drawn = cr.orders(w.run.master, model, w.draw_order, labels_of(model)) + initial = {sid: drawn[sid][0] for sid in w.draw_order[:2]} + admitted = threading.Barrier(2) + for sid, arm in initial.items(): + # Both first households must be under way before either can finish. + w.provider.specs[(model, arm, sid, 1)] = { + "check": lambda _: admitted.wait(timeout=120) + } + real_time, make_host, make_pool = time.time, w.run.host_factory, w.run.pool_factory + real_end = cr.Lanes.end_attempt started = [] readings_after = [] + state_lock = threading.Lock() + state = {"ticks": 0, "ended": 0, "recovered": False} def start_host(folder): host = make_host(folder) # Time the step from a sampled run, so a slow startup cannot hide it. - started.append(time.monotonic()) + started.append((time.monotonic(), real_time())) return host def stepped_time(): now = real_time() - if started and time.monotonic() - started[0] >= after: + with state_lock: + if not started: + return now + elapsed = time.monotonic() - started[0][0] + if elapsed < after: + return started[0][1] + elapsed readings_after.append(now) - return now - 2.0 - return now + if state["recovered"]: + return now - 2.0 + # Advance synthetic time while keeping the first overlap inside + # the two-second correction, even if this host pauses for a minute. + state["ticks"] += 1 + return started[0][1] + after - 2.0 + state["ticks"] * 1e-6 + + def start_pool(lanes): + remaining = after - (time.monotonic() - started[0][0]) + if remaining > 0: + time.sleep(remaining) + stepped_time() + w.run.host.sample() # retain both sides of the correction in real evidence + return make_pool(lanes) + + def end_attempt(lanes, sid, arm, n, *args, **kwargs): + result = real_end(lanes, sid, arm, n, *args, **kwargs) + if n == 1 and initial.get(sid) == arm: + with state_lock: + state["ended"] += 1 + state["recovered"] = state["ended"] == 2 + return result w.run.host_factory = start_host + w.run.pool_factory = start_pool + monkeypatch.setattr(cr.Lanes, "end_attempt", end_attempt) monkeypatch.setattr(time, "time", stepped_time) w.execute() assert readings_after, "the two-second wall-clock step must occur during the run" From 8355176df1659d93ce0e31dcf6d39a2aac6b2d5a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 7 Nov 2026 13:58:48 -0500 Subject: [PATCH 31/31] Hold the correction through both initial household spans before recovery. Co-Authored-By: GPT-6.1 Sol (Subfleet) Co-Authored-By: Claude Opus 5.5 --- tests/test_confirmatory_run.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index 6d635939..23443fef 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -837,7 +837,10 @@ def test_a_two_second_wall_clock_step_keeps_the_run_procedure( def start_host(folder): host = make_host(folder) # Time the step from a sampled run, so a slow startup cannot hide it. - started.append((time.monotonic(), real_time())) + # This unused pre-step reading also fixes the clock's high-water mark + # when host construction or scheduling took longer than two seconds. + before = cr.clock() + started.append((time.monotonic(), before)) return host def stepped_time(): @@ -850,7 +853,7 @@ def stepped_time(): return started[0][1] + elapsed readings_after.append(now) if state["recovered"]: - return now - 2.0 + return state["recovery_wall"] + now - state["recovery_real"] # Advance synthetic time while keeping the first overlap inside # the two-second correction, even if this host pauses for a minute. state["ticks"] += 1 @@ -866,10 +869,13 @@ def start_pool(lanes): def end_attempt(lanes, sid, arm, n, *args, **kwargs): result = real_end(lanes, sid, arm, n, *args, **kwargs) - if n == 1 and initial.get(sid) == arm: + if n == 1 and sid in initial: with state_lock: state["ended"] += 1 - state["recovered"] = state["ended"] == 2 + if state["ended"] == len(initial) * len(labels_of(model)): + state["recovery_real"] = real_time() + state["recovery_wall"] = started[0][1] + after + 0.1 + state["recovered"] = True return result w.run.host_factory = start_host