diff --git a/docs/runbook.md b/docs/runbook.md index 060c0711..9894c590 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -208,8 +208,8 @@ Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08 ([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are rows of their own, so neither row can be re-run under its name. A synchronous -re-run or row repair would not show the re-route: litellm 1.88.x sets the -response model to the requested name, which PolicyBench records as +re-run or row repair would not show the re-route: litellm (1.88.x, 1.95.0) +sets the response model to the requested name, which PolicyBench records as `provider_resolved_model`, and ignores Gemini's `modelVersion`. The Gemini batch path, which calls google-genai directly, does record `modelVersion`. diff --git a/policybench/config.py b/policybench/config.py index cdb943b7..be8d632b 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -87,8 +87,8 @@ # answers were requested on 2026-08-18, before Gemini 3.8 Flash was # released (2026-09-02), so they are Gemini 3.7 Flash, and this name can # no longer re-run them. A synchronous run, which row repairs also use, - # would not show either re-route: litellm 1.88.x sets the response model - # to the requested name and never reads Gemini's modelVersion, so + # would not show either re-route: litellm (1.88.x, 1.95.0) sets the response + # model to the requested name and never reads Gemini's modelVersion, so # provider_resolved_model records gemini-3.7-flash or gemini-3.5-flash. # The Gemini batch path (google-genai) does record modelVersion. GET # models/ still returns each retired model's own version string. diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index e4c9921b..d1724209 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -433,7 +433,19 @@ def refuse_policyengine(label: str, value) -> None: # The litellm releases whose Gemini search-tool handling # allow_gemini_search_with_functions was checked against. It was written against # 1.88.1; 1.88.6's litellm/llms/vertex_ai/gemini/ is byte-identical to 1.88.1's. -GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6"}) +# 1.95.0 only rewraps the lines of both patched methods (their syntax trees match +# 1.88.6's), and it keeps googleSearch beside functions by itself when +# include_server_side_tool_invocations is set. On a continuation turn it returns +# each server-side search part's thoughtSignature on the part Gemini put it on, +# where 1.88.x sent one signature on both parts. It also counts +# toolUsePromptTokenCount in prompt tokens when a response carries no search +# grounding. The pilot's function-calling turns carry no grounding metadata +# (see turn_searches), so on 1.95.0 any toolUsePromptTokenCount Gemini reports +# on a web-arm turn adds to its prompt tokens and cost, though Google bills +# Google Search per query and not by its tokens +# (ai.google.dev/gemini-api/docs/tool-combination). Each run's manifest records +# the litellm it ran on. +GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6", "1.95.0"}) CALCULATE_DESCRIPTION = ( "Run PolicyEngine US (policyengine-us 2.41.0) on one household and return " @@ -1230,8 +1242,11 @@ def allow_gemini_search_with_functions() -> None: ``completion`` keyword reaches it later, as a provider extra, after the search tool is gone. Gemini 3 and 4 models take both together with the flag (ai.google.dev/gemini-api/docs/tool-combination), and the web arms always - send it, so both of litellm's drops are skipped. Refuses any other litellm, - whose code this was not checked against. + send it, so both of litellm's drops are skipped. litellm 1.95.0 reads the + flag before it maps the tools and keeps the search tool by itself, so there + the patch changes nothing, and CI, which runs the locked 1.95.0, can't catch + a broken patch body; only a test run on 1.88.x exercises it. Refuses any + other litellm, whose code this was not checked against. """ version = importlib.metadata.version("litellm") if version not in GEMINI_SEARCH_PATCH_LITELLM: @@ -1467,11 +1482,11 @@ def _attach_deferred_results(self, message: dict) -> None: When Claude calls a server tool beside a client tool, the API runs the server tool on the next request, and that response begins with its - result, without the call. litellm 1.88.x re-sends a server result only - beside its call in the same message, so left in place the result drops - out of later requests, and the API refuses the call that lacks it (a - live probe got that 400 on the third request). The result joins its - call, the only place litellm sends it from. + result, without the call. litellm 1.88.x and 1.95.0 re-send a server + result only beside its call in the same message, so left in place the + result drops out of later requests, and the API refuses the call that + lacks it (a live probe got that 400 on the third request). The result + joins its call, the only place litellm sends it from. """ fields = message.get("provider_specific_fields") if not isinstance(fields, dict): diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 32076b55..bf657235 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2746,6 +2746,19 @@ def _keep_litellm_gemini_config(monkeypatch): monkeypatch.setattr(VertexGeminiConfig, name, VertexGeminiConfig.__dict__[name]) +# The tools each checked litellm keeps, unpatched, when a web arm sends the +# search tool beside functions with include_server_side_tool_invocations set. +UNPATCHED_GEMINI_TOOLS = { + "1.88.1": ["function_declarations"], + "1.88.6": ["function_declarations"], + "1.95.0": ["function_declarations", "googleSearch"], +} + + +def test_each_checked_litellm_records_what_it_does_unpatched(): + assert set(UNPATCHED_GEMINI_TOOLS) == pilot.GEMINI_SEARCH_PATCH_LITELLM + + def test_the_gemini_patch_keeps_search_beside_functions(monkeypatch): from litellm.utils import get_optional_params @@ -2764,13 +2777,149 @@ def tool_keys(): # Installed from the lock, litellm is a release the patch was checked # against (the lock test below), so this runs rather than skips. - assert importlib.metadata.version("litellm") in pilot.GEMINI_SEARCH_PATCH_LITELLM - assert tool_keys() == ["function_declarations"] # litellm drops search + version = importlib.metadata.version("litellm") + assert version in pilot.GEMINI_SEARCH_PATCH_LITELLM + assert tool_keys() == UNPATCHED_GEMINI_TOOLS[version] pilot.allow_gemini_search_with_functions() assert tool_keys() == ["function_declarations", "googleSearch"] assert calculate_tool["function"]["name"] == pilot.TOOL_NAME +# The thoughtSignatures each checked litellm sends back on the toolCall and +# toolResponse parts of a server-side search: 1.88.x merges the two parts' +# signatures into one, the call's, and 1.95.0 returns each on its own part, as +# ai.google.dev/gemini-api/docs/tool-combination asks. +RESENT_SEARCH_SIGNATURES = { + "1.88.1": ("SIG_CALL", "SIG_CALL"), + "1.88.6": ("SIG_CALL", "SIG_CALL"), + "1.95.0": ("SIG_CALL", "SIG_RESP"), +} + + +def test_each_checked_litellm_records_the_signatures_it_resends(): + assert set(RESENT_SEARCH_SIGNATURES) == pilot.GEMINI_SEARCH_PATCH_LITELLM + + +def _fill(schema): + kind = schema.get("type") + if kind == "object": + return {k: _fill(v) for k, v in schema.get("properties", {}).items()} + if kind in ("number", "integer"): + return 0 + if kind == "boolean": + return False + if kind == "array": + return [] + return "Computed value = 0" + + +def test_the_gemini_web_and_tool_arm_runs_through_the_locked_litellm(monkeypatch): + """Drives the web+pe arm's conversation through litellm's own Gemini + request and response code, with canned generateContent replies in place of + the network. The first reply searches and calls the calculator; the next + answers through the submit tool.""" + import httpx + from litellm import completion + + _keep_litellm_gemini_config(monkeypatch) + monkeypatch.setenv("GEMINI_API_KEY", "test-key") + pilot.allow_gemini_search_with_functions() + search = {"id": "srv-1", "toolType": "GOOGLE_SEARCH_WEB"} + sent = [] + + def reply(request_body): + if not sent[1:]: + parts = [ + {"thought": True, "text": "plan", "thoughtSignature": "SIG_THOUGHT"}, + { + "toolCall": {**search, "args": {"queries": ["colorado snap"]}}, + "thoughtSignature": "SIG_CALL", + }, + { + "toolResponse": { + **search, + "response": {"results": [{"url": "https://cdhs.colorado.gov"}]}, + }, + "thoughtSignature": "SIG_RESP", + }, + { + "functionCall": { + "name": pilot.TOOL_NAME, + "args": {"situation": {}, "variables": ["snap"]}, + }, + "thoughtSignature": "SIG_FN", + }, + ] + else: + (submit,) = [ + d + for t in request_body["tools"] + for d in t.get("function_declarations", []) + if d["name"] != pilot.TOOL_NAME + ] + parts = [ + { + "functionCall": { + "name": submit["name"], + "args": _fill(submit["parameters"]), + }, + "thoughtSignature": "SIG_ANSWER", + } + ] + return { + "candidates": [ + { + "content": {"role": "model", "parts": parts}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 1000, + "candidatesTokenCount": 100, + "thoughtsTokenCount": 50, + "totalTokenCount": 1150, + }, + "modelVersion": "gemini-4-argon", + } + + def send(self, request, **kwargs): + body = json.loads(request.content) + sent.append(body) + return httpx.Response(200, json=reply(body), request=request) + + monkeypatch.setattr(httpx.Client, "send", send) + record = pilot.run_household( + make_scenario(), + VARIABLES, + lambda arguments: ({"values": {"snap": 1234.0}}, 0.01), + completion, + pilot.Spend(10.0), + profile=pilot.model_profile(pilot.MODEL_NAME, "web+pe"), + ) + + assert record["status"] == "complete" and len(sent) == 2 + first, second = sent + assert sorted(k for t in first["tools"] for k in t) == [ + "function_declarations", + "googleSearch", + ] + assert first["toolConfig"]["includeServerSideToolInvocations"] is True + (model_turn,) = [c for c in second["contents"] if c["role"] == "model"] + signatures = { + key: part.get("thoughtSignature") + for part in model_turn["parts"] + for key in ("toolCall", "toolResponse", "function_call") + if key in part + } + version = importlib.metadata.version("litellm") + assert ( + signatures["toolCall"], + signatures["toolResponse"], + ) == RESENT_SEARCH_SIGNATURES[version] + assert signatures["function_call"] == "SIG_FN" + + def test_the_locked_litellm_is_one_the_gemini_patch_was_checked_against(): """A lock bump past the checked releases fails here, before a live web arm refuses to start.""" diff --git a/uv.lock b/uv.lock index 001d0ea8..8f0e80c7 100644 --- a/uv.lock +++ b/uv.lock @@ -1515,7 +1515,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.88.6" +version = "1.95.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1531,9 +1531,20 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f4/a6/d2312da4d65390f0d899a669731500983c8a10bd89009657680949c5766d/litellm-1.88.6.tar.gz", hash = "sha256:07d947dfe92f137d5296c40285dbcf97cb405b4300d73c0feda48d76468d6eb4", size = 13889078, upload-time = "2026-08-09T00:06:33.818Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/0b/2e42b95ccca2e2a8a524808e9c414f70a0352165dac589efa01192afb8ba/litellm-1.88.6-py3-none-any.whl", hash = "sha256:8c01f7e9aebded5b3509128be648518a2ec4f1feb8dfb87a539895091651898b", size = 15271205, upload-time = "2026-08-09T00:06:30.607Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/0e/96/8cdfb9aaf584b57af35a0423c111a1c1264a78b548cebbb5ed96defacdab/litellm-1.95.0.tar.gz", hash = "sha256:0ef126d52c7a559f8353e50d60fd0d5e7e6c8767ad54df25ddaf79b9edca1afc", size = 17513577, upload-time = "2026-08-02T02:52:49.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/c1/470070205a3b36c4d99d95046102468165a55bfd675f32c552566085e746/litellm-1.95.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4bafa3494d503a3c6c1f2eeb785a2af364d85463c71a92cabaaa761c9227d62a", size = 26425608, upload-time = "2026-08-02T02:52:05.277Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c1/54c79849f4b60aab55772d69099557e49c86c6a43a863d65c1bc985246b3/litellm-1.95.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4c2a06d2263a07a29228cd3af2b594cf86cd3a4182a7324c517cba88a9415969", size = 26303794, upload-time = "2026-08-02T02:52:08.675Z" }, + { url = "https://files.pythonhosted.org/packages/c5/52/31f2f50063e619b782c74df3ea8835270cce40f94dadaed350cf2f499aa0/litellm-1.95.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac37bb6d2be191bafc0ce590ed06d21dd7e5a37599ffc1af1071899f6c74b73", size = 24921269, upload-time = "2026-08-02T02:52:12.281Z" }, + { url = "https://files.pythonhosted.org/packages/33/d0/ad0272853cc450f8bb4a40a93d206e767e18ac2ff3f91870374e1d9fc090/litellm-1.95.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:cb667f84f08520f32b076e03c7a3fa51bf3f7e8b641dade34ab046bf00314d6b", size = 26421359, upload-time = "2026-08-02T02:52:16.048Z" }, + { url = "https://files.pythonhosted.org/packages/02/c1/4301aa8ef6d2fb0e4a2b8dec973d7c4499f680b26d2e1b77643864235a4b/litellm-1.95.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1bdf7153557cc0851fa9477b137fde476c56d5de92a5778ecfc6c3a75439a4e1", size = 26300401, upload-time = "2026-08-02T02:52:19.501Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3d/6cd087bd541f18d924f17bd8e1bb68a7f9d73d03274c5339f9de563bc992/litellm-1.95.0-cp312-cp312-win_amd64.whl", hash = "sha256:62cc5d834e8223dbd16c9ad0b46c73354b6d67cc7fa0eba2764ce65b3b8c474f", size = 24917446, upload-time = "2026-08-02T02:52:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/fd/47/719785f65b01779cf7568c329430c93b2e7832498deb3deec53bdd106f8d/litellm-1.95.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9d80a9adc506bfce48145621d6649e3fd428407811eb00211bcce33344054701", size = 26422310, upload-time = "2026-08-02T02:52:26.626Z" }, + { url = "https://files.pythonhosted.org/packages/55/48/06447e1125d7ae31bd24d34d2af2833b15aed79dc5f7e8b45862c1d06af8/litellm-1.95.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cf014ff515825ad49937b4cdf95616270789311db7841d16702e0a5b1ac5b067", size = 26300935, upload-time = "2026-08-02T02:52:30.032Z" }, + { url = "https://files.pythonhosted.org/packages/45/6f/388f85ebcb4e239dc738cab99307051d5a8a69d907023b0dbb200ee95226/litellm-1.95.0-cp313-cp313-win_amd64.whl", hash = "sha256:c73df441153e585832d4e90e3717d17ae888b269daa71d723336369e81ef884b", size = 24917355, upload-time = "2026-08-02T02:52:33.593Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f6/5b2e944d5c8cea5164e10a28220d9af598792a9c3f25605bcf14946323ef/litellm-1.95.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c8e712f95764a9a730f3aec9dff8be916973423411ca12a79a35c8bea3f7d5a4", size = 26423318, upload-time = "2026-08-02T02:52:37.629Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f7/d7450f5d3d7b566900067a21bdfafb5cee62bc0bc73b190e8598df1f2e67/litellm-1.95.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:071a63c0e1d949bf7ed5e5c73843cb042a6e00324d3f214b13f8c6b90da6822d", size = 26299965, upload-time = "2026-08-02T02:52:41.577Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/c2b7d8b239021c7737a228ab5ffe4f40e4a596ce4629ff3c8899bfd40904/litellm-1.95.0-cp314-cp314-win_amd64.whl", hash = "sha256:c3684dcf16aefe98bd6f11586d60a9a92bb1bf7e85468a6a4ac28a65532fab3e", size = 24920863, upload-time = "2026-08-02T02:52:45.663Z" }, ] [package.optional-dependencies]