Tool pilot: a web+pe3 arm, web search with the version 3 calculator

scripts/confirmatory_run.py

@@ -90,7 +90,7 @@
 
 The runner runs the treatment harness's code. It refuses to start unless its
 ``tool_pilot_argon.py``, ``policybench`` package and fix module are the
-treatment commit's (1c8c645b), its tree is clean at the design's run commit,
+treatment commit's (c0a2885b), its tree is clean at the design's run commit,
 the machine is on AC power with the lid open, and the runs folder holds no
 records: a run is never resumed. ``--workers`` must equal the design's
 engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the
@@ -131,7 +131,7 @@
 
 import tool_pilot_argon as pilot  # noqa: E402
 
-TREATMENT_COMMIT = "1c8c645bcd7219a76d09bb8c24e5fca51305d149"
+TREATMENT_COMMIT = "c0a2885bfc8f5df59a93458230c445befb986dcb"
 # The treatment's code, which must be the treatment commit's.
 TREATMENT_PATHS = (
     "scripts/tool_pilot_argon.py",

scripts/tool_pilot_argon.py

@@ -25,8 +25,10 @@
      computes every reference on that system). The ``oracle`` subcommand checks
      that the reference situations reproduce every scored reference through it.
      ``--arm web`` offers the provider's own web search in place of the tool,
-     and ``--arm web+pe`` offers both; rows are labeled ``<model>+web`` and
-     ``<model>+web+pe-tool``. Searches are logged per turn and priced at list
+     ``--arm web+pe`` offers both, and ``--arm web+pe3`` offers web search with
+     the version 3 calculator (pe3's notice, with web search named in its
+     opening); rows are labeled ``<model>+web``, ``<model>+web+pe-tool`` and
+     ``<model>+web+pe-tool-v3``. Searches are logged per turn and priced at list
      (SEARCH_USD_PER_1K) on top of their tokens; policybench.org and GitHub are
      blocked where the provider allows it (BLOCKED_DOMAINS), and ``score``
      writes searches.csv with an audit of any that touch the benchmark.
@@ -154,14 +156,20 @@
 # with no tools), so a prompt version other than the board's has its own
 # no-tools baseline. Repairs, retries and truncation then follow this pilot's
 # loop, not the board's; they matter only when a first answer does not parse.
-ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3")
+# "web+pe3" offers web search with the version 3 calculator (Max 2026-10-26,
+# for the Aspen preview: web search against web search and the calculator is
+# the comparison, so version 3 runs with web search too). Its notice is pe3's
+# with web search named in the opening sentence, as web+pe's is pe's.
+ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3", "web+pe3")
+WEB_ARMS = ("web", "web+pe", "web+pe3")
 ARM_SUFFIX = {
     "none": "+no-tools",
     "pe": "+pe-tool",
     "web": "+web",
     "web+pe": "+web+pe-tool",
     "pe2": "+pe-tool-v2",
     "pe3": "+pe-tool-v3",
+    "web+pe3": "+web+pe-tool-v3",
 }
 # The calculator's features, each on or off, so a study can change one at a
 # time (Max, 2026-10-23: "measuring this as an experiment could be
@@ -202,6 +210,7 @@
     "web+pe": frozenset(),
     "pe2": V2_FEATURES,
     "pe3": frozenset(TOOL_FEATURES),
+    "web+pe3": frozenset(TOOL_FEATURES),
 }
 _FEATURE_ARM = re.compile(r"^pe:([a-z]+(?:\+[a-z]+)*)$")
 
@@ -383,13 +392,21 @@ def engine_text(text: str) -> str:
 )
 # The PE notice with web search named in its opening sentence; the rest of it,
 # the tool's description and the turn cap, is unchanged.
+_WEB_CALCULATE_OPENING = (
+    "You also have web search, which searches the public web, and a "
+    "`calculate_household` function."
+)
+
+
+def with_web_opening(notice: str) -> str:
+    """A calculator notice with web search named in its opening sentence."""
+    if notice.count(_CALCULATE_OPENING) != 1:
+        raise ValueError("a calculator notice opens with the calculator once")
+    return notice.replace(_CALCULATE_OPENING, _WEB_CALCULATE_OPENING)
+
+
 WEB_TOOL_NOTICE, JSON_WEB_TOOL_NOTICE = (
-    notice.replace(
-        _CALCULATE_OPENING,
-        "You also have web search, which searches the public web, and a "
-        "`calculate_household` function.",
-    )
-    for notice in (TOOL_NOTICE, JSON_TOOL_NOTICE)
+    with_web_opening(notice) for notice in (TOOL_NOTICE, JSON_TOOL_NOTICE)
 )
 
 # Prompt versions ``run --prompt`` takes. "v1" is the board's prompt. "v2-pilot"
@@ -2423,6 +2440,8 @@ def notice(self) -> str:
             notice = WEB_NOTICE if tool else JSON_WEB_NOTICE
         elif self.arm == "web+pe":
             notice = WEB_TOOL_NOTICE if tool else JSON_WEB_TOOL_NOTICE
+        elif self.arm == "web+pe3":
+            notice = with_web_opening(compose_tool_notice(arm_features(self.arm), tool))
         elif self.arm == "pe2":
             return engine_text(TOOL_NOTICE_V2 if tool else JSON_TOOL_NOTICE_V2)
         elif self.arm != "pe" and arm_features(self.arm) is not None:
@@ -2447,7 +2466,7 @@ def tool_version(self) -> int:
 
     @property
     def uses_web(self) -> bool:
-        return self.arm in ("web", "web+pe")
+        return self.arm in WEB_ARMS
 
     @property
     def provider(self) -> str:
@@ -2497,7 +2516,7 @@ def model_profile(
             f"unknown prompt {prompt_version!r}; the prompts are "
             f"{', '.join(PROMPT_VERSIONS)}"
         )
-    if benchmark_search_rule and arm not in ("web", "web+pe"):
+    if benchmark_search_rule and arm not in WEB_ARMS:
         raise SystemExit("the benchmark-search rule needs a web arm")
     model_id = MODELS[name]
     return ModelProfile(
@@ -3106,7 +3125,8 @@ def run_household(
     tool is offered under tool_choice auto and the cap is a user message.
 
     The profile's arm sets the tools: the calculate tool ("pe"), the
-    provider's web search ("web"), or both ("web+pe"). The provider runs a
+    provider's web search ("web"), or both ("web+pe", or "web+pe3" with the
+    version 3 calculator). The provider runs a
     search inside the response that asked for it, so searches use no
     calculation turn; each response's searches are logged and their list
     price added to its cost.
@@ -5675,7 +5695,7 @@ def main(argv: list[str] | None = None) -> None:
                 default="pe",
                 help="the tools: none, PolicyEngine (pe, pe2, pe3, or pe: plus "
                 "calculator features joined by +), web search (web), or both "
-                "(web+pe)",
+                "(web+pe, or web+pe3 with the version 3 calculator)",
             )
             p.add_argument(
                 "--prompt",

tests/test_confirmatory_run.py

@@ -3322,3 +3322,31 @@ def broken(head, anchor_time):
     last = json.loads((w.runs() / "anchors.jsonl").read_text().splitlines()[-1])
     assert last["models"][model]["run-log.jsonl"]["lines"] == len(w.lines(model))
     assert last["host-log.jsonl"] == w.run.host.heads()
+
+
+def test_the_treatment_code_is_the_treatment_commits():
+    """The preflight refuses a run whose treatment code differs from the
+    treatment commit's, so this fails in CI first, while the confirmatory run
+    is pending. It comes out after that run, before v2-microcosm-batches
+    reaches main, where it would fail every unrelated change to policybench/
+    (#249, reply 6596821678)."""
+    root = Path(cr.__file__).resolve().parents[1]
+    git = ["git", "-C", str(root)]
+    if subprocess.run(
+        [*git, "cat-file", "-e", f"{cr.TREATMENT_COMMIT}^{{commit}}"]
+    ).returncode:
+        pytest.skip("this checkout lacks the treatment commit")
+    changed = subprocess.run(
+        [
+            *git,
+            "diff",
+            "--quiet",
+            cr.TREATMENT_COMMIT,
+            "HEAD",
+            "--",
+            *cr.TREATMENT_PATHS,
+        ]
+    ).returncode
+    assert changed == 0, (
+        "move TREATMENT_COMMIT to the last commit that touches TREATMENT_PATHS"
+    )

tests/test_tool_pilot_argon.py

@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 import dataclasses
+import hashlib
 import importlib.metadata
 import json
 import math
@@ -2296,6 +2297,7 @@ def test_web_notices_name_search_and_keep_the_answer_instruction():
         "web+pe": "gpt-6-sol+web+pe-tool",
         "pe2": "gpt-6-sol+pe-tool-v2",
         "pe3": "gpt-6-sol+pe-tool-v3",
+        "web+pe3": "gpt-6-sol+web+pe-tool-v3",
     }
     with pytest.raises(SystemExit):
         pilot.model_profile(SOL, "search")
@@ -3168,7 +3170,7 @@ def test_the_school_meal_wording_matches_every_reference():
 
 
 def test_the_benchmark_search_rule_rides_only_on_web_notices():
-    for arm in ("web", "web+pe"):
+    for arm in ("web", "web+pe", "web+pe3"):
         for name in (SOL, OPUS):
             with_rule = pilot.model_profile(name, arm, "v2-pilot", True)
             without = pilot.model_profile(name, arm, "v2-pilot")
@@ -3774,3 +3776,135 @@ def test_the_law_date_sentence_states_the_fallback_and_the_snap_split():
     assert "asset limits through December 2026" in prompt
     assert "2026 HHS poverty guidelines from October 2026" in prompt
     assert "fiscal year 2026 values for all of 2026" not in prompt
+
+
+def test_web_and_tool_v3_is_tool_v3_with_web_search_named_in_its_opening():
+    # web+pe3 is to pe3 what web+pe is to pe: the same calculator, features,
+    # schema and notice, with web search named in the notice's first sentence.
+    for name in (SOL, OPUS):
+        both = pilot.model_profile(name, "web+pe3")
+        alone = pilot.model_profile(name, "pe3")
+        assert both.uses_web and both.uses_calculator and not alone.uses_web
+        assert (
+            both.tool_features == alone.tool_features == frozenset(pilot.TOOL_FEATURES)
+        )
+        assert pilot.profile_treatment(both) == pilot.profile_treatment(alone)
+        assert both.notice != alone.notice and "web search" in both.notice
+        assert (
+            both.notice.split("function. ", 1)[1]
+            == alone.notice.split("function. ", 1)[1]
+        )
+        assert both.notice == alone.notice.replace(
+            pilot._CALCULATE_OPENING, pilot._WEB_CALCULATE_OPENING
+        )
+        ruled = pilot.model_profile(name, "web+pe3", "v2-pilot", True)
+        assert ruled.notice.endswith(pilot.BENCHMARK_SEARCH_RULE)
+    # web+pe's notice is unchanged by the refactor into with_web_opening.
+    assert pilot.WEB_TOOL_NOTICE == pilot.TOOL_NOTICE.replace(
+        pilot._CALCULATE_OPENING,
+        "You also have web search, which searches the public web, and a "
+        "`calculate_household` function.",
+    )
+    with pytest.raises(ValueError):
+        pilot.with_web_opening("no calculator here")
+
+
+def test_a_gemini_web_and_tool_v3_arm_offers_search_and_the_v3_calculator():
+    profile = pilot.model_profile(pilot.MODEL_NAME, "web+pe3", "v2-pilot", True)
+    model = FakeModel(
+        [
+            with_fields(
+                response([submit({"snap": 2553, VARIABLES[1]: 0})]),
+                gemini_searches(("g1", ["co snap 2026"])),
+            )
+        ]
+    )
+    record = pilot.run_household(
+        make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile
+    )
+    assert record["status"] == "complete" and record["arm"] == "web+pe3"
+    assert record["tool_features"] == sorted(pilot.TOOL_FEATURES)
+    (request,) = model.requests
+    assert request["tools"][0] == {"googleSearch": {}}
+    assert [t["function"]["name"] for t in request["tools"][1:]] == [
+        pilot.TOOL_NAME,
+        "submit_outputs",
+    ]
+    assert request["include_server_side_tool_invocations"] is True
+    assert request["messages"][0]["content"].endswith(profile.notice)
+    assert record["totals"]["web_searches"] == 1
+    assert (
+        record["calculate_tool_sha256"]
+        == pilot.profile_treatment(pilot.model_profile(pilot.MODEL_NAME, "pe3"))[
+            "calculate_tool_sha256"
+        ]
+    )
+    assert pilot.record_treatment_problem(record, profile) is None
+
+
+def test_an_anthropic_web_and_tool_v3_arm_offers_search_and_the_v3_calculator():
+    profile = pilot.model_profile(OPUS, "web+pe3", "v2-pilot", True)
+    model = FakeModel(
+        [response([], content=json_answer({"snap": 2553, VARIABLES[1]: 0}))]
+    )
+    record = pilot.run_household(
+        make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile
+    )
+    assert record["status"] == "complete" and record["arm"] == "web+pe3"
+    assert record["tool_features"] == sorted(pilot.TOOL_FEATURES)
+    (request,) = model.requests
+    assert [t.get("name") or t["function"]["name"] for t in request["tools"]] == [
+        "web_search",
+        pilot.TOOL_NAME,
+    ]
+    system = request["messages"][0]["content"]  # text blocks on Anthropic
+    assert "".join(part["text"] for part in system).endswith(profile.notice)
+    assert (
+        record["calculate_tool_sha256"]
+        == pilot.profile_treatment(pilot.model_profile(OPUS, "pe3"))[
+            "calculate_tool_sha256"
+        ]
+    )
+    assert pilot.record_treatment_problem(record, profile) is None
+
+
+def test_a_responses_web_and_tool_v3_arm_offers_search_and_the_v3_calculator():
+    profile = pilot.model_profile(SOL, "web+pe3", "v2-pilot", True)
+    model = FakeResponses(
+        [
+            responses_response(
+                [
+                    web_search_call(
+                        "ws_1", SimpleNamespace(type="search", query="co snap")
+                    ),
+                    sol_submit({"snap": 2553, VARIABLES[1]: 0}),
+                ],
+                "resp_1",
+            ),
+        ]
+    )
+    record = pilot.run_household(
+        make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile
+    )
+    assert record["status"] == "complete" and record["arm"] == "web+pe3"
+    assert record["tool_features"] == sorted(pilot.TOOL_FEATURES)
+    assert (
+        record["tool_notice_sha256"]
+        == hashlib.sha256(profile.notice.encode()).hexdigest()
+    )
+    (first,) = model.requests
+    assert first["tools"][0] == {
+        "type": "web_search",
+        "filters": {"blocked_domains": list(pilot.BLOCKED_DOMAINS)},
+    }
+    assert [t.get("name") for t in first["tools"][1:]] == [
+        pilot.TOOL_NAME,
+        "submit_outputs",
+    ]
+    assert (
+        record["calculate_tool_sha256"]
+        == pilot.profile_treatment(pilot.model_profile(SOL, "pe3"))[
+            "calculate_tool_sha256"
+        ]
+    )
+    assert pilot.record_treatment_problem(record, profile) is None