diff --git a/scripts/confirmatory_run.py b/scripts/confirmatory_run.py index 12f42ea0..d0957ef7 100644 --- a/scripts/confirmatory_run.py +++ b/scripts/confirmatory_run.py @@ -90,7 +90,7 @@ The runner runs the treatment harness's code. It refuses to start unless its ``tool_pilot_argon.py``, ``policybench`` package and fix module are the -treatment commit's (1c8c645b), its tree is clean at the design's run commit, +treatment commit's (c0a2885b), its tree is clean at the design's run commit, the machine is on AC power with the lid open, and the runs folder holds no records: a run is never resumed. ``--workers`` must equal the design's engine workers per model. An interrupt (SIGINT or SIGTERM) only notes the @@ -131,7 +131,7 @@ import tool_pilot_argon as pilot # noqa: E402 -TREATMENT_COMMIT = "1c8c645bcd7219a76d09bb8c24e5fca51305d149" +TREATMENT_COMMIT = "c0a2885bfc8f5df59a93458230c445befb986dcb" # The treatment's code, which must be the treatment commit's. TREATMENT_PATHS = ( "scripts/tool_pilot_argon.py", diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 8024413a..9b466ee8 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -25,8 +25,10 @@ computes every reference on that system). The ``oracle`` subcommand checks that the reference situations reproduce every scored reference through it. ``--arm web`` offers the provider's own web search in place of the tool, - and ``--arm web+pe`` offers both; rows are labeled ``+web`` and - ``+web+pe-tool``. Searches are logged per turn and priced at list + ``--arm web+pe`` offers both, and ``--arm web+pe3`` offers web search with + the version 3 calculator (pe3's notice, with web search named in its + opening); rows are labeled ``+web``, ``+web+pe-tool`` and + ``+web+pe-tool-v3``. Searches are logged per turn and priced at list (SEARCH_USD_PER_1K) on top of their tokens; policybench.org and GitHub are blocked where the provider allows it (BLOCKED_DOMAINS), and ``score`` writes searches.csv with an audit of any that touch the benchmark. @@ -154,7 +156,12 @@ # with no tools), so a prompt version other than the board's has its own # no-tools baseline. Repairs, retries and truncation then follow this pilot's # loop, not the board's; they matter only when a first answer does not parse. -ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3") +# "web+pe3" offers web search with the version 3 calculator (Max 2026-10-26, +# for the Aspen preview: web search against web search and the calculator is +# the comparison, so version 3 runs with web search too). Its notice is pe3's +# with web search named in the opening sentence, as web+pe's is pe's. +ARMS = ("none", "pe", "web", "web+pe", "pe2", "pe3", "web+pe3") +WEB_ARMS = ("web", "web+pe", "web+pe3") ARM_SUFFIX = { "none": "+no-tools", "pe": "+pe-tool", @@ -162,6 +169,7 @@ "web+pe": "+web+pe-tool", "pe2": "+pe-tool-v2", "pe3": "+pe-tool-v3", + "web+pe3": "+web+pe-tool-v3", } # The calculator's features, each on or off, so a study can change one at a # time (Max, 2026-10-23: "measuring this as an experiment could be @@ -202,6 +210,7 @@ "web+pe": frozenset(), "pe2": V2_FEATURES, "pe3": frozenset(TOOL_FEATURES), + "web+pe3": frozenset(TOOL_FEATURES), } _FEATURE_ARM = re.compile(r"^pe:([a-z]+(?:\+[a-z]+)*)$") @@ -383,13 +392,21 @@ def engine_text(text: str) -> str: ) # The PE notice with web search named in its opening sentence; the rest of it, # the tool's description and the turn cap, is unchanged. +_WEB_CALCULATE_OPENING = ( + "You also have web search, which searches the public web, and a " + "`calculate_household` function." +) + + +def with_web_opening(notice: str) -> str: + """A calculator notice with web search named in its opening sentence.""" + if notice.count(_CALCULATE_OPENING) != 1: + raise ValueError("a calculator notice opens with the calculator once") + return notice.replace(_CALCULATE_OPENING, _WEB_CALCULATE_OPENING) + + WEB_TOOL_NOTICE, JSON_WEB_TOOL_NOTICE = ( - notice.replace( - _CALCULATE_OPENING, - "You also have web search, which searches the public web, and a " - "`calculate_household` function.", - ) - for notice in (TOOL_NOTICE, JSON_TOOL_NOTICE) + with_web_opening(notice) for notice in (TOOL_NOTICE, JSON_TOOL_NOTICE) ) # Prompt versions ``run --prompt`` takes. "v1" is the board's prompt. "v2-pilot" @@ -2423,6 +2440,8 @@ def notice(self) -> str: notice = WEB_NOTICE if tool else JSON_WEB_NOTICE elif self.arm == "web+pe": notice = WEB_TOOL_NOTICE if tool else JSON_WEB_TOOL_NOTICE + elif self.arm == "web+pe3": + notice = with_web_opening(compose_tool_notice(arm_features(self.arm), tool)) elif self.arm == "pe2": return engine_text(TOOL_NOTICE_V2 if tool else JSON_TOOL_NOTICE_V2) elif self.arm != "pe" and arm_features(self.arm) is not None: @@ -2447,7 +2466,7 @@ def tool_version(self) -> int: @property def uses_web(self) -> bool: - return self.arm in ("web", "web+pe") + return self.arm in WEB_ARMS @property def provider(self) -> str: @@ -2497,7 +2516,7 @@ def model_profile( f"unknown prompt {prompt_version!r}; the prompts are " f"{', '.join(PROMPT_VERSIONS)}" ) - if benchmark_search_rule and arm not in ("web", "web+pe"): + if benchmark_search_rule and arm not in WEB_ARMS: raise SystemExit("the benchmark-search rule needs a web arm") model_id = MODELS[name] return ModelProfile( @@ -3106,7 +3125,8 @@ def run_household( tool is offered under tool_choice auto and the cap is a user message. The profile's arm sets the tools: the calculate tool ("pe"), the - provider's web search ("web"), or both ("web+pe"). The provider runs a + provider's web search ("web"), or both ("web+pe", or "web+pe3" with the + version 3 calculator). The provider runs a search inside the response that asked for it, so searches use no calculation turn; each response's searches are logged and their list price added to its cost. @@ -5675,7 +5695,7 @@ def main(argv: list[str] | None = None) -> None: default="pe", help="the tools: none, PolicyEngine (pe, pe2, pe3, or pe: plus " "calculator features joined by +), web search (web), or both " - "(web+pe)", + "(web+pe, or web+pe3 with the version 3 calculator)", ) p.add_argument( "--prompt", diff --git a/tests/test_confirmatory_run.py b/tests/test_confirmatory_run.py index f9c69a6f..81861839 100644 --- a/tests/test_confirmatory_run.py +++ b/tests/test_confirmatory_run.py @@ -3322,3 +3322,31 @@ def broken(head, anchor_time): last = json.loads((w.runs() / "anchors.jsonl").read_text().splitlines()[-1]) assert last["models"][model]["run-log.jsonl"]["lines"] == len(w.lines(model)) assert last["host-log.jsonl"] == w.run.host.heads() + + +def test_the_treatment_code_is_the_treatment_commits(): + """The preflight refuses a run whose treatment code differs from the + treatment commit's, so this fails in CI first, while the confirmatory run + is pending. It comes out after that run, before v2-microcosm-batches + reaches main, where it would fail every unrelated change to policybench/ + (#249, reply 6596821678).""" + root = Path(cr.__file__).resolve().parents[1] + git = ["git", "-C", str(root)] + if subprocess.run( + [*git, "cat-file", "-e", f"{cr.TREATMENT_COMMIT}^{{commit}}"] + ).returncode: + pytest.skip("this checkout lacks the treatment commit") + changed = subprocess.run( + [ + *git, + "diff", + "--quiet", + cr.TREATMENT_COMMIT, + "HEAD", + "--", + *cr.TREATMENT_PATHS, + ] + ).returncode + assert changed == 0, ( + "move TREATMENT_COMMIT to the last commit that touches TREATMENT_PATHS" + ) diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 91b41048..b27a7fe4 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -8,6 +8,7 @@ from __future__ import annotations import dataclasses +import hashlib import importlib.metadata import json import math @@ -2296,6 +2297,7 @@ def test_web_notices_name_search_and_keep_the_answer_instruction(): "web+pe": "gpt-6-sol+web+pe-tool", "pe2": "gpt-6-sol+pe-tool-v2", "pe3": "gpt-6-sol+pe-tool-v3", + "web+pe3": "gpt-6-sol+web+pe-tool-v3", } with pytest.raises(SystemExit): pilot.model_profile(SOL, "search") @@ -3168,7 +3170,7 @@ def test_the_school_meal_wording_matches_every_reference(): def test_the_benchmark_search_rule_rides_only_on_web_notices(): - for arm in ("web", "web+pe"): + for arm in ("web", "web+pe", "web+pe3"): for name in (SOL, OPUS): with_rule = pilot.model_profile(name, arm, "v2-pilot", True) without = pilot.model_profile(name, arm, "v2-pilot") @@ -3774,3 +3776,135 @@ def test_the_law_date_sentence_states_the_fallback_and_the_snap_split(): assert "asset limits through December 2026" in prompt assert "2026 HHS poverty guidelines from October 2026" in prompt assert "fiscal year 2026 values for all of 2026" not in prompt + + +def test_web_and_tool_v3_is_tool_v3_with_web_search_named_in_its_opening(): + # web+pe3 is to pe3 what web+pe is to pe: the same calculator, features, + # schema and notice, with web search named in the notice's first sentence. + for name in (SOL, OPUS): + both = pilot.model_profile(name, "web+pe3") + alone = pilot.model_profile(name, "pe3") + assert both.uses_web and both.uses_calculator and not alone.uses_web + assert ( + both.tool_features == alone.tool_features == frozenset(pilot.TOOL_FEATURES) + ) + assert pilot.profile_treatment(both) == pilot.profile_treatment(alone) + assert both.notice != alone.notice and "web search" in both.notice + assert ( + both.notice.split("function. ", 1)[1] + == alone.notice.split("function. ", 1)[1] + ) + assert both.notice == alone.notice.replace( + pilot._CALCULATE_OPENING, pilot._WEB_CALCULATE_OPENING + ) + ruled = pilot.model_profile(name, "web+pe3", "v2-pilot", True) + assert ruled.notice.endswith(pilot.BENCHMARK_SEARCH_RULE) + # web+pe's notice is unchanged by the refactor into with_web_opening. + assert pilot.WEB_TOOL_NOTICE == pilot.TOOL_NOTICE.replace( + pilot._CALCULATE_OPENING, + "You also have web search, which searches the public web, and a " + "`calculate_household` function.", + ) + with pytest.raises(ValueError): + pilot.with_web_opening("no calculator here") + + +def test_a_gemini_web_and_tool_v3_arm_offers_search_and_the_v3_calculator(): + profile = pilot.model_profile(pilot.MODEL_NAME, "web+pe3", "v2-pilot", True) + model = FakeModel( + [ + with_fields( + response([submit({"snap": 2553, VARIABLES[1]: 0})]), + gemini_searches(("g1", ["co snap 2026"])), + ) + ] + ) + record = pilot.run_household( + make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + assert record["status"] == "complete" and record["arm"] == "web+pe3" + assert record["tool_features"] == sorted(pilot.TOOL_FEATURES) + (request,) = model.requests + assert request["tools"][0] == {"googleSearch": {}} + assert [t["function"]["name"] for t in request["tools"][1:]] == [ + pilot.TOOL_NAME, + "submit_outputs", + ] + assert request["include_server_side_tool_invocations"] is True + assert request["messages"][0]["content"].endswith(profile.notice) + assert record["totals"]["web_searches"] == 1 + assert ( + record["calculate_tool_sha256"] + == pilot.profile_treatment(pilot.model_profile(pilot.MODEL_NAME, "pe3"))[ + "calculate_tool_sha256" + ] + ) + assert pilot.record_treatment_problem(record, profile) is None + + +def test_an_anthropic_web_and_tool_v3_arm_offers_search_and_the_v3_calculator(): + profile = pilot.model_profile(OPUS, "web+pe3", "v2-pilot", True) + model = FakeModel( + [response([], content=json_answer({"snap": 2553, VARIABLES[1]: 0}))] + ) + record = pilot.run_household( + make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + assert record["status"] == "complete" and record["arm"] == "web+pe3" + assert record["tool_features"] == sorted(pilot.TOOL_FEATURES) + (request,) = model.requests + assert [t.get("name") or t["function"]["name"] for t in request["tools"]] == [ + "web_search", + pilot.TOOL_NAME, + ] + system = request["messages"][0]["content"] # text blocks on Anthropic + assert "".join(part["text"] for part in system).endswith(profile.notice) + assert ( + record["calculate_tool_sha256"] + == pilot.profile_treatment(pilot.model_profile(OPUS, "pe3"))[ + "calculate_tool_sha256" + ] + ) + assert pilot.record_treatment_problem(record, profile) is None + + +def test_a_responses_web_and_tool_v3_arm_offers_search_and_the_v3_calculator(): + profile = pilot.model_profile(SOL, "web+pe3", "v2-pilot", True) + model = FakeResponses( + [ + responses_response( + [ + web_search_call( + "ws_1", SimpleNamespace(type="search", query="co snap") + ), + sol_submit({"snap": 2553, VARIABLES[1]: 0}), + ], + "resp_1", + ), + ] + ) + record = pilot.run_household( + make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + assert record["status"] == "complete" and record["arm"] == "web+pe3" + assert record["tool_features"] == sorted(pilot.TOOL_FEATURES) + assert ( + record["tool_notice_sha256"] + == hashlib.sha256(profile.notice.encode()).hexdigest() + ) + (first,) = model.requests + assert first["tools"][0] == { + "type": "web_search", + "filters": {"blocked_domains": list(pilot.BLOCKED_DOMAINS)}, + } + assert [t.get("name") for t in first["tools"][1:]] == [ + pilot.TOOL_NAME, + "submit_outputs", + ] + assert ( + record["calculate_tool_sha256"] + == pilot.profile_treatment(pilot.model_profile(SOL, "pe3"))[ + "calculate_tool_sha256" + ] + ) + assert pilot.record_treatment_problem(record, profile) is None