From c55e34aec8d32060420e4cd3e25282729e3dcfd0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 15:09:20 +0000 Subject: [PATCH 1/4] Move litellm to 1.95.0 in uv.lock (GHSA-g5ff-637f-6q2m) `uv lock --upgrade-package litellm==1.95.0`. Dependabot alert #96 covers every litellm below 1.95.0, and no 1.88.x release after 1.88.6 exists. No other package moves; pyproject.toml is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6K3qkpWxSr7Tw1Lc2m --- uv.lock | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index d7da6bde..30524004 100644 --- a/uv.lock +++ b/uv.lock @@ -1515,7 +1515,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.88.6" +version = "1.95.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1531,9 +1531,20 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f4/a6/d2312da4d65390f0d899a669731500983c8a10bd89009657680949c5766d/litellm-1.88.6.tar.gz", hash = "sha256:07d947dfe92f137d5296c40285dbcf97cb405b4300d73c0feda48d76468d6eb4", size = 13889078, upload-time = "2026-08-09T00:06:33.818Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/0b/2e42b95ccca2e2a8a524808e9c414f70a0352165dac589efa01192afb8ba/litellm-1.88.6-py3-none-any.whl", hash = "sha256:8c01f7e9aebded5b3509128be648518a2ec4f1feb8dfb87a539895091651898b", size = 15271205, upload-time = "2026-08-09T00:06:30.607Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/0e/96/8cdfb9aaf584b57af35a0423c111a1c1264a78b548cebbb5ed96defacdab/litellm-1.95.0.tar.gz", hash = "sha256:0ef126d52c7a559f8353e50d60fd0d5e7e6c8767ad54df25ddaf79b9edca1afc", size = 17513577, upload-time = "2026-08-02T02:52:49.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/c1/470070205a3b36c4d99d95046102468165a55bfd675f32c552566085e746/litellm-1.95.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4bafa3494d503a3c6c1f2eeb785a2af364d85463c71a92cabaaa761c9227d62a", size = 26425608, upload-time = "2026-08-02T02:52:05.277Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c1/54c79849f4b60aab55772d69099557e49c86c6a43a863d65c1bc985246b3/litellm-1.95.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4c2a06d2263a07a29228cd3af2b594cf86cd3a4182a7324c517cba88a9415969", size = 26303794, upload-time = "2026-08-02T02:52:08.675Z" }, + { url = "https://files.pythonhosted.org/packages/c5/52/31f2f50063e619b782c74df3ea8835270cce40f94dadaed350cf2f499aa0/litellm-1.95.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac37bb6d2be191bafc0ce590ed06d21dd7e5a37599ffc1af1071899f6c74b73", size = 24921269, upload-time = "2026-08-02T02:52:12.281Z" }, + { url = "https://files.pythonhosted.org/packages/33/d0/ad0272853cc450f8bb4a40a93d206e767e18ac2ff3f91870374e1d9fc090/litellm-1.95.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:cb667f84f08520f32b076e03c7a3fa51bf3f7e8b641dade34ab046bf00314d6b", size = 26421359, upload-time = "2026-08-02T02:52:16.048Z" }, + { url = "https://files.pythonhosted.org/packages/02/c1/4301aa8ef6d2fb0e4a2b8dec973d7c4499f680b26d2e1b77643864235a4b/litellm-1.95.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1bdf7153557cc0851fa9477b137fde476c56d5de92a5778ecfc6c3a75439a4e1", size = 26300401, upload-time = "2026-08-02T02:52:19.501Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3d/6cd087bd541f18d924f17bd8e1bb68a7f9d73d03274c5339f9de563bc992/litellm-1.95.0-cp312-cp312-win_amd64.whl", hash = "sha256:62cc5d834e8223dbd16c9ad0b46c73354b6d67cc7fa0eba2764ce65b3b8c474f", size = 24917446, upload-time = "2026-08-02T02:52:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/fd/47/719785f65b01779cf7568c329430c93b2e7832498deb3deec53bdd106f8d/litellm-1.95.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9d80a9adc506bfce48145621d6649e3fd428407811eb00211bcce33344054701", size = 26422310, upload-time = "2026-08-02T02:52:26.626Z" }, + { url = "https://files.pythonhosted.org/packages/55/48/06447e1125d7ae31bd24d34d2af2833b15aed79dc5f7e8b45862c1d06af8/litellm-1.95.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cf014ff515825ad49937b4cdf95616270789311db7841d16702e0a5b1ac5b067", size = 26300935, upload-time = "2026-08-02T02:52:30.032Z" }, + { url = "https://files.pythonhosted.org/packages/45/6f/388f85ebcb4e239dc738cab99307051d5a8a69d907023b0dbb200ee95226/litellm-1.95.0-cp313-cp313-win_amd64.whl", hash = "sha256:c73df441153e585832d4e90e3717d17ae888b269daa71d723336369e81ef884b", size = 24917355, upload-time = "2026-08-02T02:52:33.593Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f6/5b2e944d5c8cea5164e10a28220d9af598792a9c3f25605bcf14946323ef/litellm-1.95.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c8e712f95764a9a730f3aec9dff8be916973423411ca12a79a35c8bea3f7d5a4", size = 26423318, upload-time = "2026-08-02T02:52:37.629Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f7/d7450f5d3d7b566900067a21bdfafb5cee62bc0bc73b190e8598df1f2e67/litellm-1.95.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:071a63c0e1d949bf7ed5e5c73843cb042a6e00324d3f214b13f8c6b90da6822d", size = 26299965, upload-time = "2026-08-02T02:52:41.577Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/c2b7d8b239021c7737a228ab5ffe4f40e4a596ce4629ff3c8899bfd40904/litellm-1.95.0-cp314-cp314-win_amd64.whl", hash = "sha256:c3684dcf16aefe98bd6f11586d60a9a92bb1bf7e85468a6a4ac28a65532fab3e", size = 24920863, upload-time = "2026-08-02T02:52:45.663Z" }, ] [package.optional-dependencies] From 265f66b03314596a75f711c1d6ed166cff5705e5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 11:59:27 -0500 Subject: [PATCH 2/4] Certify litellm 1.95.0 for the tool pilot's Gemini search patch The lock moved to litellm 1.95.0, which the Gemini search-tool gate refused. Offline captures of every pilot arm (Gemini, Claude and GPT across pe, web and web+pe, plus Claude's deferred server results) match 1.88.6's request bodies except in one place: Gemini web+pe continuations now return each server-side search part's thoughtSignature on its own part, where 1.88.x sent the call's signature on both. - Add 1.95.0 to GEMINI_SEARCH_PATCH_LITELLM and say what changed. - Record, per checked release, which tools litellm keeps unpatched and which signatures it re-sends, and require one entry per gated release. - Run the Gemini web+pe arm end to end through litellm's own Gemini code with canned replies, and pin the turn-0 tools and the turn-1 signatures for the installed release. - Name 1.95.0 beside 1.88.x where the docs say how litellm reports the Gemini model and re-sends server results. Co-Authored-By: Claude Opus 5.5 --- docs/runbook.md | 4 +- policybench/config.py | 4 +- scripts/tool_pilot_argon.py | 25 ++++-- tests/test_tool_pilot_argon.py | 153 ++++++++++++++++++++++++++++++++- 4 files changed, 172 insertions(+), 14 deletions(-) diff --git a/docs/runbook.md b/docs/runbook.md index 4274efa0..5cc92a67 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -208,8 +208,8 @@ Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08 ([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are rows of their own, so neither row can be re-run under its name. A synchronous -re-run or row repair would not show the re-route: litellm 1.88.x sets the -response model to the requested name, which PolicyBench records as +re-run or row repair would not show the re-route: litellm (1.88.x, 1.95.0) +sets the response model to the requested name, which PolicyBench records as `provider_resolved_model`, and ignores Gemini's `modelVersion`. The Gemini batch path, which calls google-genai directly, does record `modelVersion`. diff --git a/policybench/config.py b/policybench/config.py index cdb943b7..be8d632b 100644 --- a/policybench/config.py +++ b/policybench/config.py @@ -87,8 +87,8 @@ # answers were requested on 2026-08-18, before Gemini 3.8 Flash was # released (2026-09-02), so they are Gemini 3.7 Flash, and this name can # no longer re-run them. A synchronous run, which row repairs also use, - # would not show either re-route: litellm 1.88.x sets the response model - # to the requested name and never reads Gemini's modelVersion, so + # would not show either re-route: litellm (1.88.x, 1.95.0) sets the response + # model to the requested name and never reads Gemini's modelVersion, so # provider_resolved_model records gemini-3.7-flash or gemini-3.5-flash. # The Gemini batch path (google-genai) does record modelVersion. GET # models/ still returns each retired model's own version string. diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index e4c9921b..a0fbf680 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -433,7 +433,14 @@ def refuse_policyengine(label: str, value) -> None: # The litellm releases whose Gemini search-tool handling # allow_gemini_search_with_functions was checked against. It was written against # 1.88.1; 1.88.6's litellm/llms/vertex_ai/gemini/ is byte-identical to 1.88.1's. -GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6"}) +# 1.95.0 only rewraps the lines of both patched methods (their syntax trees match +# 1.88.6's), and it keeps googleSearch beside functions by itself when +# include_server_side_tool_invocations is set. On a continuation turn it returns +# each server-side search part's thoughtSignature on the part Gemini put it on, +# where 1.88.x sent one signature on both parts. It also counts +# toolUsePromptTokenCount in prompt tokens when a response carries no search +# grounding. Each run's manifest records the litellm it ran on. +GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6", "1.95.0"}) CALCULATE_DESCRIPTION = ( "Run PolicyEngine US (policyengine-us 2.41.0) on one household and return " @@ -1230,8 +1237,10 @@ def allow_gemini_search_with_functions() -> None: ``completion`` keyword reaches it later, as a provider extra, after the search tool is gone. Gemini 3 and 4 models take both together with the flag (ai.google.dev/gemini-api/docs/tool-combination), and the web arms always - send it, so both of litellm's drops are skipped. Refuses any other litellm, - whose code this was not checked against. + send it, so both of litellm's drops are skipped. litellm 1.95.0 reads the + flag before it maps the tools and keeps the search tool by itself, so there + the patch changes nothing. Refuses any other litellm, whose code this was + not checked against. """ version = importlib.metadata.version("litellm") if version not in GEMINI_SEARCH_PATCH_LITELLM: @@ -1467,11 +1476,11 @@ def _attach_deferred_results(self, message: dict) -> None: When Claude calls a server tool beside a client tool, the API runs the server tool on the next request, and that response begins with its - result, without the call. litellm 1.88.x re-sends a server result only - beside its call in the same message, so left in place the result drops - out of later requests, and the API refuses the call that lacks it (a - live probe got that 400 on the third request). The result joins its - call, the only place litellm sends it from. + result, without the call. litellm 1.88.x and 1.95.0 re-send a server + result only beside its call in the same message, so left in place the + result drops out of later requests, and the API refuses the call that + lacks it (a live probe got that 400 on the third request). The result + joins its call, the only place litellm sends it from. """ fields = message.get("provider_specific_fields") if not isinstance(fields, dict): diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 32076b55..ecf35287 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2746,6 +2746,19 @@ def _keep_litellm_gemini_config(monkeypatch): monkeypatch.setattr(VertexGeminiConfig, name, VertexGeminiConfig.__dict__[name]) +# The tools each checked litellm keeps, unpatched, when a web arm sends the +# search tool beside functions with include_server_side_tool_invocations set. +UNPATCHED_GEMINI_TOOLS = { + "1.88.1": ["function_declarations"], + "1.88.6": ["function_declarations"], + "1.95.0": ["function_declarations", "googleSearch"], +} + + +def test_each_checked_litellm_records_what_it_does_unpatched(): + assert set(UNPATCHED_GEMINI_TOOLS) == pilot.GEMINI_SEARCH_PATCH_LITELLM + + def test_the_gemini_patch_keeps_search_beside_functions(monkeypatch): from litellm.utils import get_optional_params @@ -2764,13 +2777,149 @@ def tool_keys(): # Installed from the lock, litellm is a release the patch was checked # against (the lock test below), so this runs rather than skips. - assert importlib.metadata.version("litellm") in pilot.GEMINI_SEARCH_PATCH_LITELLM - assert tool_keys() == ["function_declarations"] # litellm drops search + version = importlib.metadata.version("litellm") + assert version in pilot.GEMINI_SEARCH_PATCH_LITELLM + assert tool_keys() == UNPATCHED_GEMINI_TOOLS[version] pilot.allow_gemini_search_with_functions() assert tool_keys() == ["function_declarations", "googleSearch"] assert calculate_tool["function"]["name"] == pilot.TOOL_NAME +# The thoughtSignatures each checked litellm sends back on the toolCall and +# toolResponse parts of a server-side search: 1.88.x merges the two parts' +# signatures into one, the call's, and 1.95.0 returns each on its own part, as +# ai.google.dev/gemini-api/docs/thought-signatures asks. +RESENT_SEARCH_SIGNATURES = { + "1.88.1": ("SIG_CALL", "SIG_CALL"), + "1.88.6": ("SIG_CALL", "SIG_CALL"), + "1.95.0": ("SIG_CALL", "SIG_RESP"), +} + + +def test_each_checked_litellm_records_the_signatures_it_resends(): + assert set(RESENT_SEARCH_SIGNATURES) == pilot.GEMINI_SEARCH_PATCH_LITELLM + + +def _fill(schema): + kind = schema.get("type") + if kind == "object": + return {k: _fill(v) for k, v in schema.get("properties", {}).items()} + if kind in ("number", "integer"): + return 0 + if kind == "boolean": + return False + if kind == "array": + return [] + return "Computed value = 0" + + +def test_the_gemini_web_and_tool_arm_runs_through_the_locked_litellm(monkeypatch): + """Drives the web+pe arm's conversation through litellm's own Gemini + request and response code, with canned generateContent replies in place of + the network. The first reply searches and calls the calculator; the next + answers through the submit tool.""" + import httpx + from litellm import completion + + _keep_litellm_gemini_config(monkeypatch) + monkeypatch.setenv("GEMINI_API_KEY", "test-key") + pilot.allow_gemini_search_with_functions() + search = {"id": "srv-1", "toolType": "GOOGLE_SEARCH_WEB"} + sent = [] + + def reply(request_body): + if not sent[1:]: + parts = [ + {"thought": True, "text": "plan", "thoughtSignature": "SIG_THOUGHT"}, + { + "toolCall": {**search, "args": {"queries": ["colorado snap"]}}, + "thoughtSignature": "SIG_CALL", + }, + { + "toolResponse": { + **search, + "response": {"results": [{"url": "https://cdhs.colorado.gov"}]}, + }, + "thoughtSignature": "SIG_RESP", + }, + { + "functionCall": { + "name": pilot.TOOL_NAME, + "args": {"situation": {}, "variables": ["snap"]}, + }, + "thoughtSignature": "SIG_FN", + }, + ] + else: + (submit,) = [ + d + for t in request_body["tools"] + for d in t.get("function_declarations", []) + if d["name"] != pilot.TOOL_NAME + ] + parts = [ + { + "functionCall": { + "name": submit["name"], + "args": _fill(submit["parameters"]), + }, + "thoughtSignature": "SIG_ANSWER", + } + ] + return { + "candidates": [ + { + "content": {"role": "model", "parts": parts}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 1000, + "candidatesTokenCount": 100, + "thoughtsTokenCount": 50, + "totalTokenCount": 1150, + }, + "modelVersion": "gemini-4-argon", + } + + def send(self, request, **kwargs): + body = json.loads(request.content) + sent.append(body) + return httpx.Response(200, json=reply(body), request=request) + + monkeypatch.setattr(httpx.Client, "send", send) + record = pilot.run_household( + make_scenario(), + VARIABLES, + lambda arguments: ({"values": {"snap": 1234.0}}, 0.01), + completion, + pilot.Spend(10.0), + profile=pilot.model_profile(pilot.MODEL_NAME, "web+pe"), + ) + + assert record["status"] == "complete" and len(sent) == 2 + first, second = sent + assert sorted(k for t in first["tools"] for k in t) == [ + "function_declarations", + "googleSearch", + ] + assert first["toolConfig"]["includeServerSideToolInvocations"] is True + (model_turn,) = [c for c in second["contents"] if c["role"] == "model"] + signatures = { + key: part.get("thoughtSignature") + for part in model_turn["parts"] + for key in ("toolCall", "toolResponse", "function_call") + if key in part + } + version = importlib.metadata.version("litellm") + assert ( + signatures["toolCall"], + signatures["toolResponse"], + ) == RESENT_SEARCH_SIGNATURES[version] + assert signatures["function_call"] == "SIG_FN" + + def test_the_locked_litellm_is_one_the_gemini_patch_was_checked_against(): """A lock bump past the checked releases fails here, before a live web arm refuses to start.""" From dc43c517f09a5daf3f2a5e7dc53beb173c479b66 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 14:25:23 -0500 Subject: [PATCH 3/4] Correct the Gemini signature citation and note 1.95.0's tool-use tokens The test comment now cites the tool-combination guide, which asks a client that keeps its own history to send back each tool call's and result's signature; the thought-signatures page covers only the Interactions API. The litellm comment adds that the pilot's function-calling turns carry no grounding metadata, so on 1.95.0 any toolUsePromptTokenCount Gemini reports there adds to prompt tokens and cost, while Google bills Google Search per query. The patch docstring says only a test run on 1.88.x exercises the patch body. Comments and one docstring only. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 12 +++++++++--- tests/test_tool_pilot_argon.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index a0fbf680..fd729708 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -439,7 +439,12 @@ def refuse_policyengine(label: str, value) -> None: # each server-side search part's thoughtSignature on the part Gemini put it on, # where 1.88.x sent one signature on both parts. It also counts # toolUsePromptTokenCount in prompt tokens when a response carries no search -# grounding. Each run's manifest records the litellm it ran on. +# grounding. The pilot's function-calling turns carry no grounding metadata +# (see bill_searches), so on 1.95.0 any toolUsePromptTokenCount Gemini reports +# on a web-arm turn adds to its prompt tokens and cost, though Google bills +# Google Search per query and not by its tokens +# (ai.google.dev/gemini-api/docs/tool-combination). Each run's manifest records +# the litellm it ran on. GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6", "1.95.0"}) CALCULATE_DESCRIPTION = ( @@ -1239,8 +1244,9 @@ def allow_gemini_search_with_functions() -> None: (ai.google.dev/gemini-api/docs/tool-combination), and the web arms always send it, so both of litellm's drops are skipped. litellm 1.95.0 reads the flag before it maps the tools and keeps the search tool by itself, so there - the patch changes nothing. Refuses any other litellm, whose code this was - not checked against. + the patch changes nothing, and CI, which runs the locked 1.95.0, can't catch + a broken patch body; only a test run on 1.88.x exercises it. Refuses any + other litellm, whose code this was not checked against. """ version = importlib.metadata.version("litellm") if version not in GEMINI_SEARCH_PATCH_LITELLM: diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index ecf35287..bf657235 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2788,7 +2788,7 @@ def tool_keys(): # The thoughtSignatures each checked litellm sends back on the toolCall and # toolResponse parts of a server-side search: 1.88.x merges the two parts' # signatures into one, the call's, and 1.95.0 returns each on its own part, as -# ai.google.dev/gemini-api/docs/thought-signatures asks. +# ai.google.dev/gemini-api/docs/tool-combination asks. RESENT_SEARCH_SIGNATURES = { "1.88.1": ("SIG_CALL", "SIG_CALL"), "1.88.6": ("SIG_CALL", "SIG_CALL"), From 77ebb5e9c2e158649554f05fff9cb14826a7a9b4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 15:07:09 -0500 Subject: [PATCH 4/4] Cite turn_searches for the grounding probe (review r2) The comment on GEMINI_SEARCH_PATCH_LITELLM pointed at bill_searches, but the live probe that found no grounding metadata on function-calling turns sits in turn_searches's docstring; bill_searches calls turn_searches. The change touches only a comment, so the module's AST stays the same. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index fd729708..d1724209 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -440,7 +440,7 @@ def refuse_policyengine(label: str, value) -> None: # where 1.88.x sent one signature on both parts. It also counts # toolUsePromptTokenCount in prompt tokens when a response carries no search # grounding. The pilot's function-calling turns carry no grounding metadata -# (see bill_searches), so on 1.95.0 any toolUsePromptTokenCount Gemini reports +# (see turn_searches), so on 1.95.0 any toolUsePromptTokenCount Gemini reports # on a web-arm turn adds to its prompt tokens and cost, though Google bills # Google Search per query and not by its tokens # (ai.google.dev/gemini-api/docs/tool-combination). Each run's manifest records