Move litellm to 1.95.0 (GHSA-g5ff-637f-6q2m) and accept it in the tool pilot's Gemini patch

docs/runbook.md

@@ -208,8 +208,8 @@ Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08
 ([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under
 those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are
 rows of their own, so neither row can be re-run under its name. A synchronous
-re-run or row repair would not show the re-route: litellm 1.88.x sets the
-response model to the requested name, which PolicyBench records as
+re-run or row repair would not show the re-route: litellm (1.88.x, 1.95.0)
+sets the response model to the requested name, which PolicyBench records as
 `provider_resolved_model`, and ignores Gemini's `modelVersion`. The Gemini
 batch path, which calls google-genai directly, does record `modelVersion`.
 

policybench/config.py

@@ -87,8 +87,8 @@
     # answers were requested on 2026-08-18, before Gemini 3.8 Flash was
     # released (2026-09-02), so they are Gemini 3.7 Flash, and this name can
     # no longer re-run them. A synchronous run, which row repairs also use,
-    # would not show either re-route: litellm 1.88.x sets the response model
-    # to the requested name and never reads Gemini's modelVersion, so
+    # would not show either re-route: litellm (1.88.x, 1.95.0) sets the response
+    # model to the requested name and never reads Gemini's modelVersion, so
     # provider_resolved_model records gemini-3.7-flash or gemini-3.5-flash.
     # The Gemini batch path (google-genai) does record modelVersion. GET
     # models/<name> still returns each retired model's own version string.

scripts/tool_pilot_argon.py

@@ -433,7 +433,19 @@ def refuse_policyengine(label: str, value) -> None:
 # The litellm releases whose Gemini search-tool handling
 # allow_gemini_search_with_functions was checked against. It was written against
 # 1.88.1; 1.88.6's litellm/llms/vertex_ai/gemini/ is byte-identical to 1.88.1's.
-GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6"})
+# 1.95.0 only rewraps the lines of both patched methods (their syntax trees match
+# 1.88.6's), and it keeps googleSearch beside functions by itself when
+# include_server_side_tool_invocations is set. On a continuation turn it returns
+# each server-side search part's thoughtSignature on the part Gemini put it on,
+# where 1.88.x sent one signature on both parts. It also counts
+# toolUsePromptTokenCount in prompt tokens when a response carries no search
+# grounding. The pilot's function-calling turns carry no grounding metadata
+# (see turn_searches), so on 1.95.0 any toolUsePromptTokenCount Gemini reports
+# on a web-arm turn adds to its prompt tokens and cost, though Google bills
+# Google Search per query and not by its tokens
+# (ai.google.dev/gemini-api/docs/tool-combination). Each run's manifest records
+# the litellm it ran on.
+GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6", "1.95.0"})
 
 CALCULATE_DESCRIPTION = (
     "Run PolicyEngine US (policyengine-us 2.41.0) on one household and return "
@@ -1230,8 +1242,11 @@ def allow_gemini_search_with_functions() -> None:
     ``completion`` keyword reaches it later, as a provider extra, after the
     search tool is gone. Gemini 3 and 4 models take both together with the flag
     (ai.google.dev/gemini-api/docs/tool-combination), and the web arms always
-    send it, so both of litellm's drops are skipped. Refuses any other litellm,
-    whose code this was not checked against.
+    send it, so both of litellm's drops are skipped. litellm 1.95.0 reads the
+    flag before it maps the tools and keeps the search tool by itself, so there
+    the patch changes nothing, and CI, which runs the locked 1.95.0, can't catch
+    a broken patch body; only a test run on 1.88.x exercises it. Refuses any
+    other litellm, whose code this was not checked against.
     """
     version = importlib.metadata.version("litellm")
     if version not in GEMINI_SEARCH_PATCH_LITELLM:
@@ -1467,11 +1482,11 @@ def _attach_deferred_results(self, message: dict) -> None:
 
         When Claude calls a server tool beside a client tool, the API runs the
         server tool on the next request, and that response begins with its
-        result, without the call. litellm 1.88.x re-sends a server result only
-        beside its call in the same message, so left in place the result drops
-        out of later requests, and the API refuses the call that lacks it (a
-        live probe got that 400 on the third request). The result joins its
-        call, the only place litellm sends it from.
+        result, without the call. litellm 1.88.x and 1.95.0 re-send a server
+        result only beside its call in the same message, so left in place the
+        result drops out of later requests, and the API refuses the call that
+        lacks it (a live probe got that 400 on the third request). The result
+        joins its call, the only place litellm sends it from.
         """
         fields = message.get("provider_specific_fields")
         if not isinstance(fields, dict):

tests/test_tool_pilot_argon.py

@@ -2746,6 +2746,19 @@ def _keep_litellm_gemini_config(monkeypatch):
         monkeypatch.setattr(VertexGeminiConfig, name, VertexGeminiConfig.__dict__[name])
 
 
+# The tools each checked litellm keeps, unpatched, when a web arm sends the
+# search tool beside functions with include_server_side_tool_invocations set.
+UNPATCHED_GEMINI_TOOLS = {
+    "1.88.1": ["function_declarations"],
+    "1.88.6": ["function_declarations"],
+    "1.95.0": ["function_declarations", "googleSearch"],
+}
+
+
+def test_each_checked_litellm_records_what_it_does_unpatched():
+    assert set(UNPATCHED_GEMINI_TOOLS) == pilot.GEMINI_SEARCH_PATCH_LITELLM
+
+
 def test_the_gemini_patch_keeps_search_beside_functions(monkeypatch):
     from litellm.utils import get_optional_params
 
@@ -2764,13 +2777,149 @@ def tool_keys():
 
     # Installed from the lock, litellm is a release the patch was checked
     # against (the lock test below), so this runs rather than skips.
-    assert importlib.metadata.version("litellm") in pilot.GEMINI_SEARCH_PATCH_LITELLM
-    assert tool_keys() == ["function_declarations"]  # litellm drops search
+    version = importlib.metadata.version("litellm")
+    assert version in pilot.GEMINI_SEARCH_PATCH_LITELLM
+    assert tool_keys() == UNPATCHED_GEMINI_TOOLS[version]
     pilot.allow_gemini_search_with_functions()
     assert tool_keys() == ["function_declarations", "googleSearch"]
     assert calculate_tool["function"]["name"] == pilot.TOOL_NAME
 
 
+# The thoughtSignatures each checked litellm sends back on the toolCall and
+# toolResponse parts of a server-side search: 1.88.x merges the two parts'
+# signatures into one, the call's, and 1.95.0 returns each on its own part, as
+# ai.google.dev/gemini-api/docs/tool-combination asks.
+RESENT_SEARCH_SIGNATURES = {
+    "1.88.1": ("SIG_CALL", "SIG_CALL"),
+    "1.88.6": ("SIG_CALL", "SIG_CALL"),
+    "1.95.0": ("SIG_CALL", "SIG_RESP"),
+}
+
+
+def test_each_checked_litellm_records_the_signatures_it_resends():
+    assert set(RESENT_SEARCH_SIGNATURES) == pilot.GEMINI_SEARCH_PATCH_LITELLM
+
+
+def _fill(schema):
+    kind = schema.get("type")
+    if kind == "object":
+        return {k: _fill(v) for k, v in schema.get("properties", {}).items()}
+    if kind in ("number", "integer"):
+        return 0
+    if kind == "boolean":
+        return False
+    if kind == "array":
+        return []
+    return "Computed value = 0"
+
+
+def test_the_gemini_web_and_tool_arm_runs_through_the_locked_litellm(monkeypatch):
+    """Drives the web+pe arm's conversation through litellm's own Gemini
+    request and response code, with canned generateContent replies in place of
+    the network. The first reply searches and calls the calculator; the next
+    answers through the submit tool."""
+    import httpx
+    from litellm import completion
+
+    _keep_litellm_gemini_config(monkeypatch)
+    monkeypatch.setenv("GEMINI_API_KEY", "test-key")
+    pilot.allow_gemini_search_with_functions()
+    search = {"id": "srv-1", "toolType": "GOOGLE_SEARCH_WEB"}
+    sent = []
+
+    def reply(request_body):
+        if not sent[1:]:
+            parts = [
+                {"thought": True, "text": "plan", "thoughtSignature": "SIG_THOUGHT"},
+                {
+                    "toolCall": {**search, "args": {"queries": ["colorado snap"]}},
+                    "thoughtSignature": "SIG_CALL",
+                },
+                {
+                    "toolResponse": {
+                        **search,
+                        "response": {"results": [{"url": "https://cdhs.colorado.gov"}]},
+                    },
+                    "thoughtSignature": "SIG_RESP",
+                },
+                {
+                    "functionCall": {
+                        "name": pilot.TOOL_NAME,
+                        "args": {"situation": {}, "variables": ["snap"]},
+                    },
+                    "thoughtSignature": "SIG_FN",
+                },
+            ]
+        else:
+            (submit,) = [
+                d
+                for t in request_body["tools"]
+                for d in t.get("function_declarations", [])
+                if d["name"] != pilot.TOOL_NAME
+            ]
+            parts = [
+                {
+                    "functionCall": {
+                        "name": submit["name"],
+                        "args": _fill(submit["parameters"]),
+                    },
+                    "thoughtSignature": "SIG_ANSWER",
+                }
+            ]
+        return {
+            "candidates": [
+                {
+                    "content": {"role": "model", "parts": parts},
+                    "finishReason": "STOP",
+                    "index": 0,
+                }
+            ],
+            "usageMetadata": {
+                "promptTokenCount": 1000,
+                "candidatesTokenCount": 100,
+                "thoughtsTokenCount": 50,
+                "totalTokenCount": 1150,
+            },
+            "modelVersion": "gemini-4-argon",
+        }
+
+    def send(self, request, **kwargs):
+        body = json.loads(request.content)
+        sent.append(body)
+        return httpx.Response(200, json=reply(body), request=request)
+
+    monkeypatch.setattr(httpx.Client, "send", send)
+    record = pilot.run_household(
+        make_scenario(),
+        VARIABLES,
+        lambda arguments: ({"values": {"snap": 1234.0}}, 0.01),
+        completion,
+        pilot.Spend(10.0),
+        profile=pilot.model_profile(pilot.MODEL_NAME, "web+pe"),
+    )
+
+    assert record["status"] == "complete" and len(sent) == 2
+    first, second = sent
+    assert sorted(k for t in first["tools"] for k in t) == [
+        "function_declarations",
+        "googleSearch",
+    ]
+    assert first["toolConfig"]["includeServerSideToolInvocations"] is True
+    (model_turn,) = [c for c in second["contents"] if c["role"] == "model"]
+    signatures = {
+        key: part.get("thoughtSignature")
+        for part in model_turn["parts"]
+        for key in ("toolCall", "toolResponse", "function_call")
+        if key in part
+    }
+    version = importlib.metadata.version("litellm")
+    assert (
+        signatures["toolCall"],
+        signatures["toolResponse"],
+    ) == RESENT_SEARCH_SIGNATURES[version]
+    assert signatures["function_call"] == "SIG_FN"
+
+
 def test_the_locked_litellm_is_one_the_gemini_patch_was_checked_against():
     """A lock bump past the checked releases fails here, before a live web arm
     refuses to start."""