From 294e98fe4a9c70fd251e13445122f2a5386f5829 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 16:06:32 -0700 Subject: [PATCH 1/3] Tool pilot: a web+pe3 arm, web search with the version 3 calculator The Aspen preview's comparison is web search against web search and the calculator, and the calculator's version 3 had run only without web search. web+pe3 is to pe3 what web+pe is to pe: the same seven features, schema and notice, with web search named in the notice's opening sentence, and the benchmark-search rule allowed. WEB_ARMS names the web arms in one place; with_web_opening builds both web+calculator notices and refuses a notice that does not open with the calculator once (web+pe's text is unchanged, which a test checks). Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 35 ++++++++++++---- tests/test_tool_pilot_argon.py | 77 +++++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 10 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index f09e0b6f..a6c85051 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -154,7 +154,12 @@ # with no tools), so a prompt version other than the board's has its own # no-tools baseline. Repairs, retries and truncation then follow this pilot's # loop, not the board's; they matter only when a first answer does not parse. -ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3") +# "web+pe3" offers web search with the version 3 calculator (Max 2026-10-26, +# for the Aspen preview: web search against web search and the calculator is +# the comparison, so version 3 runs with web search too). Its notice is pe3's +# with web search named in the opening sentence, as web+pe's is pe's. +ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3", "web+pe3") +WEB_ARMS = ("web", "web+pe", "web+pe3") ARM_SUFFIX = { "none": "+no-tools", "pe": "+pe-tool", @@ -162,6 +167,7 @@ "web+pe": "+web+pe-tool", "pe2": "+pe-tool-v2", "pe3": "+pe-tool-v3", + "web+pe3": "+web+pe-tool-v3", } # The calculator's features, each on or off, so a study can change one at a # time (Max, 2026-10-23: "measuring this as an experiment could be @@ -202,6 +208,7 @@ "web+pe": frozenset(), "pe2": V2_FEATURES, "pe3": frozenset(TOOL_FEATURES), + "web+pe3": frozenset(TOOL_FEATURES), } _FEATURE_ARM = re.compile(r"^pe:([a-z]+(?:\+[a-z]+)*)$") @@ -383,13 +390,21 @@ def engine_text(text: str) -> str: ) # The PE notice with web search named in its opening sentence; the rest of it, # the tool's description and the turn cap, is unchanged. +_WEB_CALCULATE_OPENING = ( + "You also have web search, which searches the public web, and a " + "`calculate_household` function." +) + + +def with_web_opening(notice: str) -> str: + """A calculator notice with web search named in its opening sentence.""" + if notice.count(_CALCULATE_OPENING) != 1: + raise ValueError("a calculator notice opens with the calculator once") + return notice.replace(_CALCULATE_OPENING, _WEB_CALCULATE_OPENING) + + WEB_TOOL_NOTICE, JSON_WEB_TOOL_NOTICE = ( - notice.replace( - _CALCULATE_OPENING, - "You also have web search, which searches the public web, and a " - "`calculate_household` function.", - ) - for notice in (TOOL_NOTICE, JSON_TOOL_NOTICE) + with_web_opening(notice) for notice in (TOOL_NOTICE, JSON_TOOL_NOTICE) ) # Prompt versions ``run --prompt`` takes. "v1" is the board's prompt. "v2-pilot" @@ -2332,6 +2347,8 @@ def notice(self) -> str: notice = WEB_NOTICE if tool else JSON_WEB_NOTICE elif self.arm == "web+pe": notice = WEB_TOOL_NOTICE if tool else JSON_WEB_TOOL_NOTICE + elif self.arm == "web+pe3": + notice = with_web_opening(compose_tool_notice(arm_features(self.arm), tool)) elif self.arm == "pe2": return engine_text(TOOL_NOTICE_V2 if tool else JSON_TOOL_NOTICE_V2) elif self.arm != "pe" and arm_features(self.arm) is not None: @@ -2356,7 +2373,7 @@ def tool_version(self) -> int: @property def uses_web(self) -> bool: - return self.arm in ("web", "web+pe") + return self.arm in WEB_ARMS @property def provider(self) -> str: @@ -2406,7 +2423,7 @@ def model_profile( f"unknown prompt {prompt_version!r}; the prompts are " f"{', '.join(PROMPT_VERSIONS)}" ) - if benchmark_search_rule and arm not in ("web", "web+pe"): + if benchmark_search_rule and arm not in WEB_ARMS: raise SystemExit("the benchmark-search rule needs a web arm") model_id = MODELS[name] return ModelProfile( diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 4e586374..2f6eb18b 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -7,6 +7,7 @@ from __future__ import annotations +import hashlib import importlib.metadata import json import math @@ -2295,6 +2296,7 @@ def test_web_notices_name_search_and_keep_the_answer_instruction(): "web+pe": "gpt-6-sol+web+pe-tool", "pe2": "gpt-6-sol+pe-tool-v2", "pe3": "gpt-6-sol+pe-tool-v3", + "web+pe3": "gpt-6-sol+web+pe-tool-v3", } with pytest.raises(SystemExit): pilot.model_profile(SOL, "search") @@ -3161,7 +3163,7 @@ def test_the_school_meal_wording_matches_every_reference(): def test_the_benchmark_search_rule_rides_only_on_web_notices(): - for arm in ("web", "web+pe"): + for arm in ("web", "web+pe", "web+pe3"): for name in (SOL, OPUS): with_rule = pilot.model_profile(name, arm, "v2-pilot", True) without = pilot.model_profile(name, arm, "v2-pilot") @@ -3598,3 +3600,76 @@ def test_the_law_date_sentence_states_the_fallback_and_the_snap_split(): assert "asset limits through December 2026" in prompt assert "2026 HHS poverty guidelines from October 2026" in prompt assert "fiscal year 2026 values for all of 2026" not in prompt + + +def test_web_and_tool_v3_is_tool_v3_with_web_search_named_in_its_opening(): + # web+pe3 is to pe3 what web+pe is to pe: the same calculator, features, + # schema and notice, with web search named in the notice's first sentence. + for name in (SOL, OPUS): + both = pilot.model_profile(name, "web+pe3") + alone = pilot.model_profile(name, "pe3") + assert both.uses_web and both.uses_calculator and not alone.uses_web + assert ( + both.tool_features == alone.tool_features == frozenset(pilot.TOOL_FEATURES) + ) + assert pilot.profile_treatment(both) == pilot.profile_treatment(alone) + assert both.notice != alone.notice and "web search" in both.notice + assert ( + both.notice.split("function. ", 1)[1] + == alone.notice.split("function. ", 1)[1] + ) + assert both.notice == alone.notice.replace( + pilot._CALCULATE_OPENING, pilot._WEB_CALCULATE_OPENING + ) + ruled = pilot.model_profile(name, "web+pe3", "v2-pilot", True) + assert ruled.notice.endswith(pilot.BENCHMARK_SEARCH_RULE) + # web+pe's notice is unchanged by the refactor into with_web_opening. + assert pilot.WEB_TOOL_NOTICE == pilot.TOOL_NOTICE.replace( + pilot._CALCULATE_OPENING, + "You also have web search, which searches the public web, and a " + "`calculate_household` function.", + ) + with pytest.raises(ValueError): + pilot.with_web_opening("no calculator here") + + +def test_a_responses_web_and_tool_v3_arm_offers_search_and_the_v3_calculator(): + profile = pilot.model_profile(SOL, "web+pe3", "v2-pilot", True) + model = FakeResponses( + [ + responses_response( + [ + web_search_call( + "ws_1", SimpleNamespace(type="search", query="co snap") + ), + sol_submit({"snap": 2553, VARIABLES[1]: 0}), + ], + "resp_1", + ), + ] + ) + record = pilot.run_household( + make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + assert record["status"] == "complete" and record["arm"] == "web+pe3" + assert record["tool_features"] == sorted(pilot.TOOL_FEATURES) + assert ( + record["tool_notice_sha256"] + == hashlib.sha256(profile.notice.encode()).hexdigest() + ) + (first,) = model.requests + assert first["tools"][0] == { + "type": "web_search", + "filters": {"blocked_domains": list(pilot.BLOCKED_DOMAINS)}, + } + assert [t.get("name") for t in first["tools"][1:]] == [ + pilot.TOOL_NAME, + "submit_outputs", + ] + assert ( + record["calculate_tool_sha256"] + == pilot.profile_treatment(pilot.model_profile(SOL, "pe3"))[ + "calculate_tool_sha256" + ] + ) + assert pilot.record_treatment_problem(record, profile) is None From ba290236729634f809efb19149a1babb2f227699 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 18:17:06 -0700 Subject: [PATCH 2/3] Tool pilot: name web+pe3 in the help text, and test it on Gemini and Anthropic Review of 294e98fe (Opus 5.5): --arm help, the module docstring and run_household's docstring now name web+pe3 and its label. Two tests send the arm through the chat path, Gemini (googleSearch beside the v3 calculator, server-side tool invocations on) and Anthropic (web_search beside it), and check the notice, the calculator schema's hash against pe3's and the record's treatment. Both fail when web+pe3 loses web search. tool_version stays 1 for pe3 and web+pe3: nothing reads it, and the arm and tool_features identify the treatment. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 11 ++++--- tests/test_tool_pilot_argon.py | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index a6c85051..3fe32fb0 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -25,8 +25,10 @@ computes every reference on that system). The ``oracle`` subcommand checks that the reference situations reproduce every scored reference through it. ``--arm web`` offers the provider's own web search in place of the tool, - and ``--arm web+pe`` offers both; rows are labeled ``+web`` and - ``+web+pe-tool``. Searches are logged per turn and priced at list + ``--arm web+pe`` offers both, and ``--arm web+pe3`` offers web search with + the version 3 calculator (pe3's notice, with web search named in its + opening); rows are labeled ``+web``, ``+web+pe-tool`` and + ``+web+pe-tool-v3``. Searches are logged per turn and priced at list (SEARCH_USD_PER_1K) on top of their tokens; policybench.org and GitHub are blocked where the provider allows it (BLOCKED_DOMAINS), and ``score`` writes searches.csv with an audit of any that touch the benchmark. @@ -3029,7 +3031,8 @@ def run_household( tool is offered under tool_choice auto and the cap is a user message. The profile's arm sets the tools: the calculate tool ("pe"), the - provider's web search ("web"), or both ("web+pe"). The provider runs a + provider's web search ("web"), or both ("web+pe", or "web+pe3" with the + version 3 calculator). The provider runs a search inside the response that asked for it, so searches use no calculation turn; each response's searches are logged and their list price added to its cost. @@ -5598,7 +5601,7 @@ def main(argv: list[str] | None = None) -> None: default="pe", help="the tools: none, PolicyEngine (pe, pe2, pe3, or pe: plus " "calculator features joined by +), web search (web), or both " - "(web+pe)", + "(web+pe, or web+pe3 with the version 3 calculator)", ) p.add_argument( "--prompt", diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 2f6eb18b..dd38289e 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -3633,6 +3633,65 @@ def test_web_and_tool_v3_is_tool_v3_with_web_search_named_in_its_opening(): pilot.with_web_opening("no calculator here") +def test_a_gemini_web_and_tool_v3_arm_offers_search_and_the_v3_calculator(): + profile = pilot.model_profile(pilot.MODEL_NAME, "web+pe3", "v2-pilot", True) + model = FakeModel( + [ + with_fields( + response([submit({"snap": 2553, VARIABLES[1]: 0})]), + gemini_searches(("g1", ["co snap 2026"])), + ) + ] + ) + record = pilot.run_household( + make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + assert record["status"] == "complete" and record["arm"] == "web+pe3" + assert record["tool_features"] == sorted(pilot.TOOL_FEATURES) + (request,) = model.requests + assert request["tools"][0] == {"googleSearch": {}} + assert [t["function"]["name"] for t in request["tools"][1:]] == [ + pilot.TOOL_NAME, + "submit_outputs", + ] + assert request["include_server_side_tool_invocations"] is True + assert request["messages"][0]["content"].endswith(profile.notice) + assert record["totals"]["web_searches"] == 1 + assert ( + record["calculate_tool_sha256"] + == pilot.profile_treatment(pilot.model_profile(pilot.MODEL_NAME, "pe3"))[ + "calculate_tool_sha256" + ] + ) + assert pilot.record_treatment_problem(record, profile) is None + + +def test_an_anthropic_web_and_tool_v3_arm_offers_search_and_the_v3_calculator(): + profile = pilot.model_profile(OPUS, "web+pe3", "v2-pilot", True) + model = FakeModel( + [response([], content=json_answer({"snap": 2553, VARIABLES[1]: 0}))] + ) + record = pilot.run_household( + make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + assert record["status"] == "complete" and record["arm"] == "web+pe3" + assert record["tool_features"] == sorted(pilot.TOOL_FEATURES) + (request,) = model.requests + assert [t.get("name") or t["function"]["name"] for t in request["tools"]] == [ + "web_search", + pilot.TOOL_NAME, + ] + system = request["messages"][0]["content"] # text blocks on Anthropic + assert "".join(part["text"] for part in system).endswith(profile.notice) + assert ( + record["calculate_tool_sha256"] + == pilot.profile_treatment(pilot.model_profile(OPUS, "pe3"))[ + "calculate_tool_sha256" + ] + ) + assert pilot.record_treatment_problem(record, profile) is None + + def test_a_responses_web_and_tool_v3_arm_offers_search_and_the_v3_calculator(): profile = pilot.model_profile(SOL, "web+pe3", "v2-pilot", True) model = FakeResponses( From 07ee27fbc3e490eef12c0c17bda8fe3686c0bfba Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 18:29:05 -0500 Subject: [PATCH 3/3] Confirmatory runner: pin the treatment to c0a2885b, which adds web+pe3 The merge c0a2885b is now the last commit that touches the treatment's code (tool_pilot_argon.py gains the web+pe3 arm), so TREATMENT_COMMIT moves there from 1c8c645b, in the constant and the module docstring. Max ruled on 2026-11-03 (d2229, amended) that the confirmatory run adds a pre-registered web vs web+pe3 stage for GPT-6 Luna, Gemini 3.5 Flash-Lite and Claude Sonnet 5.5, so its harness needs this arm. No other arm's treatment changes (#246 checks each byte for byte against 6b00a6f0). The new test runs the preflight's own comparison, so a commit that changes the treatment's code without moving the pin fails in CI, not only at run time (#249's delta review, nit 4). It comes out after the confirmatory run, before this line reaches main (#249, reply 6596821678). Co-Authored-By: Claude Opus 5.5 --- scripts/confirmatory_run.py | 4 ++-- tests/test_confirmatory_run.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 12f42ea0..d0957ef7 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -90,7 +90,7 @@ The runner runs the treatment harness's code. It refuses to start unless its ``tool_pilot_argon.py``, ``policybench`` package and fix module are the -treatment commit's (1c8c645b), its tree is clean at the design's run commit, +treatment commit's (c0a2885b), its tree is clean at the design's run commit, the machine is on AC power with the lid open, and the runs folder holds no records: a run is never resumed. ``--workers`` must equal the design's engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the @@ -131,7 +131,7 @@ import tool_pilot_argon as pilot # noqa: E402 -TREATMENT_COMMIT = "1c8c645bcd7219a76d09bb8c24e5fca51305d149" +TREATMENT_COMMIT = "c0a2885bfc8f5df59a93458230c445befb986dcb" # The treatment's code, which must be the treatment commit's. TREATMENT_PATHS = ( "scripts/tool_pilot_argon.py", diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index f9c69a6f..81861839 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -3322,3 +3322,31 @@ def broken(head, anchor_time): last = json.loads((w.runs() / "anchors.jsonl").read_text().splitlines()[-1]) assert last["models"][model]["run-log.jsonl"]["lines"] == len(w.lines(model)) assert last["host-log.jsonl"] == w.run.host.heads() + + +def test_the_treatment_code_is_the_treatment_commits(): + """The preflight refuses a run whose treatment code differs from the + treatment commit's, so this fails in CI first, while the confirmatory run + is pending. It comes out after that run, before v2-microcosm-batches + reaches main, where it would fail every unrelated change to policybench/ + (#249, reply 6596821678).""" + root = Path(cr.__file__).resolve().parents[1] + git = ["git", "-C", str(root)] + if subprocess.run( + [*git, "cat-file", "-e", f"{cr.TREATMENT_COMMIT}^{{commit}}"] + ).returncode: + pytest.skip("this checkout lacks the treatment commit") + changed = subprocess.run( + [ + *git, + "diff", + "--quiet", + cr.TREATMENT_COMMIT, + "HEAD", + "--", + *cr.TREATMENT_PATHS, + ] + ).returncode + assert changed == 0, ( + "move TREATMENT_COMMIT to the last commit that touches TREATMENT_PATHS" + )