From 9970c150f04d4a27bf49817c62e775b7b365f4f9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 23:56:32 -0800 Subject: [PATCH] feat(agents): shipped agents report cache reads and writes as Usage fields, not provider_details Each shipped agent (Pydantic AI, OpenHands, pi, Hermes, browser-use) now fills Usage's cache_read_tokens, cache_write_tokens and input_includes_cache, and its input_tokens counts the cache where the agent can tell: pi and Hermes add back the cache their runtimes take out of the prompt; Pydantic AI and OpenHands decide call by call, adding Anthropic's cache keys to a prompt smaller than them and leaving the flag unknown when the counts can't say. A model call that returned no usage, as Claude's streamed answers through a LiteLLM proxy, leaves the counts unknown and is counted in provider_details.unreported_calls instead of reading as zero tokens. provider_details.cache_read_tokens and cache_write_tokens move to the Usage fields for all five agents, and browser-use's cost to Usage.cost_usd. The agents' and the proxy's protocol floor is 0.1.597, the first whose Usage has the cache fields. Co-Authored-By: Claude Opus 5.5 --- .../agents/agents/browser-use/a2a_server.py | 9 +++- .../agents/browser-use/requirements.txt | 6 +-- .../agents/agents/hermes/a2a_server.py | 25 +++++++--- .../agents/agents/hermes/requirements.txt | 6 +-- .../agents/openhands/openhands_agent.py | 38 ++++++++++++++- .../agents/openhands/openhands_runtime.py | 33 ++++--------- .../agents/agents/openhands/requirements.txt | 6 +-- src/agent_env/agents/agents/pi/pi_agent.py | 23 +++++++-- .../agents/agents/pi/requirements.txt | 6 +-- .../agents/agents/proxy/requirements.txt | 6 +-- .../agents/pydantic-ai/pydantic_ai_agent.py | 46 +++++++++++++++++- .../agents/pydantic-ai/pydantic_ai_runtime.py | 30 +++++------- .../agents/pydantic-ai/requirements.txt | 6 +-- .../cli/run_bundle_openhands_test.py | 47 +++++++++++++++++++ .../cli/run_bundle_pydantic_ai_test.py | 42 +++++++++++++++++ .../browser_use/test_browser_use_agent.py | 3 +- .../agents/hermes/test_message_contract.py | 22 +++++++++ .../agents/openhands/test_openhands_agent.py | 35 ++++++++++++++ tst/unit/agents/pi/test_pi_agent.py | 32 ++++++++++++- .../pydantic_ai/test_pydantic_ai_agent.py | 36 ++++++++++++++ tst/util/fake_openai.py | 17 ++++--- 21 files changed, 389 insertions(+), 85 deletions(-) diff --git a/src/agent_env/agents/agents/browser-use/a2a_server.py b/src/agent_env/agents/agents/browser-use/a2a_server.py index eaef8ea8f..3146f8910 100644 --- a/src/agent_env/agents/agents/browser-use/a2a_server.py +++ b/src/agent_env/agents/agents/browser-use/a2a_server.py @@ -161,9 +161,14 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult: input_tokens=usage["input"], output_tokens=usage["output"], total_tokens=usage["input"] + usage["output"], + # browser-use counts the cache in its input on both of its routes: Anthropic's Messages, whose cache + # it adds in, and OpenAI's Chat Completions, whose prompt already holds it. + cache_read_tokens=usage["cache_read"], + cache_write_tokens=usage["cache_write"], + input_includes_cache=True, + cost_usd=result.get("cost_usd"), provider_details={"steps": result["steps"], "model": result["model"], "requests": usage["requests"], - "cache_read_tokens": usage["cache_read"], "cache_write_tokens": usage["cache_write"], - "cost_usd": result.get("cost_usd"), "success": result.get("success")}, + "success": result.get("success")}, )) .build() ) diff --git a/src/agent_env/agents/agents/browser-use/requirements.txt b/src/agent_env/agents/agents/browser-use/requirements.txt index 91e032ab6..dd31306bb 100644 --- a/src/agent_env/agents/agents/browser-use/requirements.txt +++ b/src/agent_env/agents/agents/browser-use/requirements.txt @@ -1,4 +1,4 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 diff --git a/src/agent_env/agents/agents/hermes/a2a_server.py b/src/agent_env/agents/agents/hermes/a2a_server.py index ee0565539..4f994a7d7 100644 --- a/src/agent_env/agents/agents/hermes/a2a_server.py +++ b/src/agent_env/agents/agents/hermes/a2a_server.py @@ -519,7 +519,10 @@ async def swap() -> None: def _usage(result) -> Usage: - """The turn's tool calls and the token counts and cost Hermes reported for its main model calls.""" + """The turn's tool calls and the token counts and cost Hermes reported for its main model calls, with + ``input_tokens`` counting the cache: Hermes' ``input_tokens`` is the prompt less its cache reads and writes, so + they're added back. Calls that reported no counts at all got none from their endpoint, as streamed Claude answers + through a LiteLLM proxy, so the turn's counts are unknown rather than nothing.""" report = result.usage or {} def count(name: str) -> int | None: @@ -527,13 +530,23 @@ def count(name: str) -> int | None: return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None cost = report.get("estimated_cost_usd") - details = {key: report[key] for key in ("api_calls", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens", - "cost_status", "turn_exit_reason") if report.get(key) is not None} + details = {key: report[key] for key in ("api_calls", "reasoning_tokens", "cost_status", "turn_exit_reason") + if report.get(key) is not None} + counts = {name: count(name) for name in ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", + "cache_write_tokens")} + if (count("api_calls") or 0) > 0 and not any(counts.values()): + return Usage(tool_call_count=result.tool_call_count, + provider_details={**details, "unreported_calls": details["api_calls"]}) + read, write = counts["cache_read_tokens"], counts["cache_write_tokens"] + input_tokens = counts["input_tokens"] return Usage( tool_call_count=result.tool_call_count, - input_tokens=count("input_tokens"), - output_tokens=count("output_tokens"), - total_tokens=count("total_tokens"), + input_tokens=None if input_tokens is None else input_tokens + (read or 0) + (write or 0), + output_tokens=counts["output_tokens"], + total_tokens=counts["total_tokens"], + cache_read_tokens=read, + cache_write_tokens=write, + input_includes_cache=None if input_tokens is None else True, cost_usd=cost if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0 else None, provider_details=details, ) diff --git a/src/agent_env/agents/agents/hermes/requirements.txt b/src/agent_env/agents/agents/hermes/requirements.txt index b0dd0ae3f..219eb7912 100644 --- a/src/agent_env/agents/agents/hermes/requirements.txt +++ b/src/agent_env/agents/agents/hermes/requirements.txt @@ -1,5 +1,5 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 jsonschema>=4.18,<5 diff --git a/src/agent_env/agents/agents/openhands/openhands_agent.py b/src/agent_env/agents/agents/openhands/openhands_agent.py index fecf02c4b..43d4f5bdb 100644 --- a/src/agent_env/agents/agents/openhands/openhands_agent.py +++ b/src/agent_env/agents/agents/openhands/openhands_agent.py @@ -17,7 +17,7 @@ import uuid from collections.abc import AsyncIterator, Callable, Mapping from contextlib import asynccontextmanager, suppress -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, Final, Literal, Protocol, Self from urllib.parse import urlsplit @@ -157,6 +157,42 @@ class Outcome: error_type: Literal["agent_error", "infra_error"] = "agent_error" +def turn_usage(calls: list[tuple[Any, bool]], *, responses: int, tool_calls: int, cost: float | None) -> Usage: + """A turn's usage from the ``TokenUsage`` OpenHands recorded for each model call, beside whether its LLM speaks + OpenAI's Responses API, with ``input_tokens`` counting the cache. A Responses call's input counts it. A Chat + Completions call's may or may not: LiteLLM reads cached tokens from OpenAI's ``prompt_tokens_details``, whose + prompt counts them, and from Anthropic's keys beside a prompt that doesn't, as a LiteLLM proxy sends Claude's. + The cache is added when the prompt is smaller than it, and the turn's flag is left unknown when the counts can't + say. OpenHands records no ``TokenUsage`` for a response without tokens, so the turn's ``responses`` beyond its + calls are counted in ``unreported_calls``.""" + counted = [_counted(usage.prompt_tokens, usage.completion_tokens, usage.cache_read_tokens, + usage.cache_write_tokens, True if responses_api else None) for usage, responses_api in calls] + unreported = responses - len(calls) + details = {"reasoning_tokens": sum(usage.reasoning_tokens or 0 for usage, _ in calls), + **({"unreported_calls": unreported} if unreported else {})} + return replace(_summed(counted, tool_calls, details), cost_usd=cost) + + +def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]: + """A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the + cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0.""" + if counts_cache or read + write == 0: + return prompt, output, read, write, 1 + if prompt < read + write: + return prompt + read + write, output, read, write, 1 + return prompt, output, read, write, 0 + + +def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage: + """The calls' counts summed: none at all when no call reported usage, the flag unknown when a call's can't be.""" + if not calls: + return Usage(tool_call_count=tool_calls, provider_details=details) + input_tokens, output_tokens, read, write, settled = (sum(column) for column in zip(*calls)) + return Usage(tool_call_count=tool_calls, input_tokens=input_tokens, output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, cache_read_tokens=read, cache_write_tokens=write, + input_includes_cache=True if settled == len(calls) else None, provider_details=details) + + @dataclass(slots=True) class _ConversationLock: lock: asyncio.Lock = field(default_factory=asyncio.Lock) diff --git a/src/agent_env/agents/agents/openhands/openhands_runtime.py b/src/agent_env/agents/agents/openhands/openhands_runtime.py index 9add55f5b..b99d3a1dc 100644 --- a/src/agent_env/agents/agents/openhands/openhands_runtime.py +++ b/src/agent_env/agents/agents/openhands/openhands_runtime.py @@ -15,14 +15,13 @@ from openhands.sdk.event import ActionEvent, Event from openhands.sdk.event.conversation_error import ConversationErrorEvent from openhands.sdk.event.error_classification import FailureKind -from openhands.sdk.llm.utils.metrics import Metrics from openhands.sdk.mcp import MCPServer from openhands.sdk.skills.skill import load_skills_from_dir from openhands.tools.preset.default import get_default_condenser, get_default_tools from pydantic import PrivateAttr from conversation_run import ConversationRun -from openhands_agent import Outcome, Turn +from openhands_agent import Outcome, Turn, turn_usage # The AgentEnv endpoint is a LiteLLM gateway; this provider prefix sends LiteLLM's requests to it as is. _PROVIDER: Final[str] = "litellm_proxy" @@ -88,29 +87,13 @@ def _agent(turn: Turn) -> Agent: def _usage(agent: Agent, tool_calls: int) -> Usage: - """What the turn's LLMs have spent: they are made for the turn, so they start at nothing.""" - llms = (agent.llm, getattr(agent.condenser, "llm", None)) - metrics = Metrics() - for each in {id(llm.metrics): llm.metrics for llm in llms if llm is not None}.values(): - metrics.merge(each) - - def spent(name: str) -> int: - return getattr(metrics.accumulated_token_usage, name, 0) or 0 - - input_tokens, output_tokens = spent("prompt_tokens"), spent("completion_tokens") - cost = metrics.accumulated_cost - return Usage( - tool_call_count=tool_calls, - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=input_tokens + output_tokens, - cost_usd=cost or None, - provider_details={ - "cache_read_tokens": spent("cache_read_tokens"), - "cache_write_tokens": spent("cache_write_tokens"), - "reasoning_tokens": spent("reasoning_tokens"), - }, - ) + """What the turn's LLMs have spent, call by call: they are made for the turn, so they start at nothing. Each + response they got has a latency recorded, with or without usage.""" + llms = {id(llm.metrics): llm for llm in (agent.llm, getattr(agent.condenser, "llm", None)) if llm is not None} + calls = [(usage, llm.uses_responses_api()) for llm in llms.values() for usage in llm.metrics.token_usages] + responses = sum(len(llm.metrics.response_latencies) for llm in llms.values()) + cost = sum(llm.metrics.accumulated_cost for llm in llms.values()) + return turn_usage(calls, responses=responses, tool_calls=tool_calls, cost=cost or None) def _unstarted(exc: Exception) -> Outcome: diff --git a/src/agent_env/agents/agents/openhands/requirements.txt b/src/agent_env/agents/agents/openhands/requirements.txt index de2a01407..4a3a05316 100644 --- a/src/agent_env/agents/agents/openhands/requirements.txt +++ b/src/agent_env/agents/agents/openhands/requirements.txt @@ -1,6 +1,6 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 openhands-sdk==1.58.2 openhands-tools==1.58.2 diff --git a/src/agent_env/agents/agents/pi/pi_agent.py b/src/agent_env/agents/agents/pi/pi_agent.py index 46e550d8c..cf11b2ede 100644 --- a/src/agent_env/agents/agents/pi/pi_agent.py +++ b/src/agent_env/agents/agents/pi/pi_agent.py @@ -255,20 +255,35 @@ def _record(run: _Run, event: dict[str, Any]) -> None: def _usage(run: _Run) -> Usage: + """The turn's usage, summed over its assistant messages, with ``input_tokens`` counting the cache: pi's ``input`` + is the prompt less its cache reads and writes, so they're added back. A message whose usage is all zero got none + from its endpoint, as a streamed Claude answer through a LiteLLM proxy, and is counted in ``unreported_calls`` + instead of as nothing used.""" totals = {"input": 0, "output": 0, "totalTokens": 0, "cacheRead": 0, "cacheWrite": 0} - cost = 0.0 + cost, reported, unreported = 0.0, 0, 0 for message in run.assistant_messages: usage = message.get("usage") or {} + counts = {key: int(usage.get(key) or 0) for key in totals} + if not any(counts.values()): + unreported += 1 + continue + reported += 1 for key in totals: - totals[key] += int(usage.get(key) or 0) + totals[key] += counts[key] cost += float((usage.get("cost") or {}).get("total") or 0) + details = {"unreported_calls": unreported} if unreported else {} + if not reported: + return Usage(tool_call_count=run.tool_calls, provider_details=details) return Usage( tool_call_count=run.tool_calls, - input_tokens=totals["input"], + input_tokens=totals["input"] + totals["cacheRead"] + totals["cacheWrite"], output_tokens=totals["output"], total_tokens=totals["totalTokens"], + cache_read_tokens=totals["cacheRead"], + cache_write_tokens=totals["cacheWrite"], + input_includes_cache=True, cost_usd=cost or None, - provider_details={"cache_read_tokens": totals["cacheRead"], "cache_write_tokens": totals["cacheWrite"]}, + provider_details=details, ) diff --git a/src/agent_env/agents/agents/pi/requirements.txt b/src/agent_env/agents/agents/pi/requirements.txt index 91e032ab6..dd31306bb 100644 --- a/src/agent_env/agents/agents/pi/requirements.txt +++ b/src/agent_env/agents/agents/pi/requirements.txt @@ -1,4 +1,4 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 diff --git a/src/agent_env/agents/agents/proxy/requirements.txt b/src/agent_env/agents/agents/proxy/requirements.txt index 91e032ab6..cbb1ff189 100644 --- a/src/agent_env/agents/agents/proxy/requirements.txt +++ b/src/agent_env/agents/agents/proxy/requirements.txt @@ -1,4 +1,4 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose proxy reports each turn's model usage, +# as the driver records it; raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 diff --git a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py index 6596f606a..d14b2831f 100644 --- a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py +++ b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py @@ -14,7 +14,7 @@ import tarfile import tempfile import uuid -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Callable, Iterable, Mapping from contextlib import asynccontextmanager, suppress from dataclasses import dataclass, field from pathlib import Path @@ -165,6 +165,50 @@ class Outcome: error_type: Literal["agent_error", "infra_error"] = "agent_error" +def turn_usage(responses: Iterable[Any], *, tool_calls: int, requests: int) -> Usage: + """A turn's usage from the usage Pydantic AI kept of each model response (its ``RequestUsage``), with + ``input_tokens`` counting the cache. Pydantic AI reads its cache counts from OpenAI's ``prompt_tokens_details``, + whose prompt counts them; a LiteLLM proxy's Claude answers send Anthropic's ``cache_read_input_tokens`` and + ``cache_creation_input_tokens`` beside a prompt that doesn't, which Pydantic AI keeps in ``details``. Those are + added when the prompt is smaller than them, and leave the turn's flag unknown otherwise, since the counts can't say. + A response that kept no usage at all, as a streamed Claude answer through that proxy, is counted in + ``unreported_calls`` instead of as nothing used.""" + calls, unreported = [], 0 + for usage in responses: + if not (usage.input_tokens or usage.output_tokens): + unreported += 1 + continue + details = usage.details or {} + if usage.cache_read_tokens or usage.cache_write_tokens: + read, write, counts_cache = usage.cache_read_tokens, usage.cache_write_tokens, True + else: + read = details.get("cache_read_input_tokens", 0) + write = details.get("cache_creation_input_tokens", 0) + counts_cache = None + calls.append(_counted(usage.input_tokens, usage.output_tokens, read, write, counts_cache)) + return _summed(calls, tool_calls, {"requests": requests, **({"unreported_calls": unreported} if unreported else {})}) + + +def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]: + """A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the + cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0.""" + if counts_cache or read + write == 0: + return prompt, output, read, write, 1 + if prompt < read + write: + return prompt + read + write, output, read, write, 1 + return prompt, output, read, write, 0 + + +def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage: + """The calls' counts summed: none at all when no call reported usage, the flag unknown when a call's can't be.""" + if not calls: + return Usage(tool_call_count=tool_calls, provider_details=details) + input_tokens, output_tokens, read, write, settled = (sum(column) for column in zip(*calls)) + return Usage(tool_call_count=tool_calls, input_tokens=input_tokens, output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, cache_read_tokens=read, cache_write_tokens=write, + input_includes_cache=True if settled == len(calls) else None, provider_details=details) + + @dataclass(slots=True) class _ConversationLock: lock: asyncio.Lock = field(default_factory=asyncio.Lock) diff --git a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py index d96e065f7..0f256a89d 100644 --- a/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py +++ b/src/agent_env/agents/agents/pydantic-ai/pydantic_ai_runtime.py @@ -11,7 +11,7 @@ from pydantic_ai.capabilities import LocalWorkspace from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UnexpectedModelBehavior, UsageLimitExceeded from pydantic_ai.mcp import MCPToolset -from pydantic_ai.messages import BinaryContent, ModelMessage, ModelMessagesTypeAdapter, UserContent +from pydantic_ai.messages import BinaryContent, ModelMessage, ModelMessagesTypeAdapter, ModelResponse, UserContent from pydantic_ai.models.openai import OpenAIChatModel from pydantic_ai.providers.openai import OpenAIProvider from pydantic_ai.providers.vllm import VLLMProvider @@ -20,7 +20,7 @@ from pydantic_ai_harness import Skills from pydantic_ai_harness.coder import Coder -from pydantic_ai_agent import MESSAGES, Outcome, Turn, to_completion +from pydantic_ai_agent import MESSAGES, Outcome, Turn, to_completion, turn_usage # Consecutive retries a tool gets for arguments it refuses, as many as Coder gives its file tools, so a model that # fumbles a command twice doesn't end the task. @@ -85,18 +85,10 @@ def _events(messages: Sequence[ModelMessage]) -> list[dict[str, Any]]: return ModelMessagesTypeAdapter.dump_python(list(messages), mode="json") -def _usage(usage: RunUsage) -> Usage: - return Usage( - tool_call_count=usage.tool_calls, - input_tokens=usage.input_tokens, - output_tokens=usage.output_tokens, - total_tokens=usage.input_tokens + usage.output_tokens, - provider_details={ - "requests": usage.requests, - "cache_read_tokens": usage.cache_read_tokens, - "cache_write_tokens": usage.cache_write_tokens, - }, - ) +def _usage(messages: Sequence[ModelMessage], usage: RunUsage) -> Usage: + """The turn's usage from each of its model responses', and its tool calls and requests from the run's.""" + return turn_usage((message.usage for message in messages if isinstance(message, ModelResponse)), + tool_calls=usage.tool_calls, requests=usage.requests) def _failure(exc: Exception, usage: Usage) -> Outcome: @@ -145,7 +137,8 @@ async def _run(self, turn: Turn) -> Outcome: for image in turn.images: prompt.append(BinaryContent(data=await asyncio.to_thread(image.path.read_bytes), media_type=image.media_type)) agent = _agent(turn) - recorded = len(history) + started = recorded = len(history) + messages: list[ModelMessage] = list(history) usage = RunUsage() try: async with agent.iter(prompt, message_history=history, @@ -158,13 +151,14 @@ async def _run(self, turn: Turn) -> Outcome: usage = run.usage result = run.result except asyncio.CancelledError: - turn.on_usage(_usage(usage)) + turn.on_usage(_usage(messages[started:], usage)) raise except Exception as exc: # every failure is the task's, typed by whose it was - return _failure(exc, _usage(usage)) + return _failure(exc, _usage(messages[started:], usage)) messages = result.all_messages() for event in _events(messages[recorded:]): turn.on_event(event) # A turn whose run finished is saved whole even if it is cancelled meanwhile, before its locks are let go. await to_completion(_save, turn, messages) - return Outcome(result.output.strip() or "The agent finished without a reply.", _usage(result.usage)) + return Outcome(result.output.strip() or "The agent finished without a reply.", + _usage(messages[started:], result.usage)) diff --git a/src/agent_env/agents/agents/pydantic-ai/requirements.txt b/src/agent_env/agents/agents/pydantic-ai/requirements.txt index 9d88ef579..1f44eca03 100644 --- a/src/agent_env/agents/agents/pydantic-ai/requirements.txt +++ b/src/agent_env/agents/agents/pydantic-ai/requirements.txt @@ -1,6 +1,6 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that takes the native trajectory format -# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.588 +# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's +# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.597 pydantic-ai-slim[mcp,openai]==2.61.0 pydantic-ai-harness[coder,skills]==0.61.0 diff --git a/tst/integration/cli/run_bundle_openhands_test.py b/tst/integration/cli/run_bundle_openhands_test.py index e1e260e49..6957cf1f1 100644 --- a/tst/integration/cli/run_bundle_openhands_test.py +++ b/tst/integration/cli/run_bundle_openhands_test.py @@ -8,12 +8,15 @@ import subprocess import time import uuid +from collections import deque from typing import Final import httpx import pytest from click.testing import CliRunner +from agentenv_protocol.a2a_agent import Usage + from agent_env.a2a_agent import protocol from agent_env.artifact.store import reset_artifact_store from agent_env.cli import cli @@ -32,6 +35,16 @@ AGENT: Final[str] = "@local/agentenv-framework/agents/openhands" MODEL: Final[str] = "fake-model-1" ECHO_ENV: Final[str] = "@local/agentenv-framework/echo-mcp/echo" +# A tool call and an answer from Claude through a LiteLLM proxy, whose prompt leaves out the cache it wrote and read, +# and an answer from GPT through its Chat Completions, whose prompt counts the cache. +CLAUDE_CALLS: Final[list[dict]] = [ + {"prompt_tokens": 10, "completion_tokens": 129, "total_tokens": 139, "cache_creation_input_tokens": 10363, + "cache_read_input_tokens": 0}, + {"prompt_tokens": 13, "completion_tokens": 43, "total_tokens": 56, "cache_creation_input_tokens": 199, + "cache_read_input_tokens": 10363}, +] +GPT_CALL: Final[dict] = {"prompt_tokens": 2303, "completion_tokens": 2, "total_tokens": 2305, + "prompt_tokens_details": {"cached_tokens": 2048}} @pytest.fixture @@ -224,6 +237,40 @@ async def steps(context): assert second["text"].startswith("OpenHands could not start the turn: "), outcomes +def test_a_turns_usage_counts_the_cache_in_its_input_and_is_unknown_where_no_usage_came_back(state, endpoint): + outcomes = {} + endpoint.respond = answer("pong") + + async def steps(context): + await DeployAgentTaskStep(id="agent", version=None, sandbox_type="local", a2a_agent_id=AGENT).execute(context) + (agent,) = context.deployed_agents + base_url = agent.a2a_url or agent.api_url + await protocol.post_agent_config(f"{base_url}/ext/agent-config", {"model": MODEL}, headers=agent.control_headers) + claude = deque(CLAUDE_CALLS) + endpoint.respond = tool_then_answer("terminal", {"command": "echo hi", "security_risk": "LOW"}, + prefix="the shell said: ") + endpoint.usage = lambda body: claude.popleft() if claude else None + outcomes["claude"] = await _send(base_url, "ctx", "Say hi with the shell.") + endpoint.respond, endpoint.usage = answer("pong"), lambda body: GPT_CALL + outcomes["gpt"] = await _send(base_url, "ctx", "Say pong.") + endpoint.usage = lambda body: None + outcomes["none"] = await _send(base_url, "ctx", "Say pong again.") + + _with_agent(state, steps) + + assert {outcome["state"] for outcome in outcomes.values()} == {"completed"}, outcomes + usage = {name: Usage.from_dict(outcome["data"]["usage"]) for name, outcome in outcomes.items()} + # GPT's prompt is as large as its cache, so whether it counts it is left unknown. + assert usage == { + "claude": Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, + provider_details={"reasoning_tokens": 0}), + "gpt": Usage(tool_call_count=0, input_tokens=2303, output_tokens=2, total_tokens=2305, cache_read_tokens=2048, + cache_write_tokens=0, input_includes_cache=None, provider_details={"reasoning_tokens": 0}), + "none": Usage(tool_call_count=0, provider_details={"reasoning_tokens": 0, "unreported_calls": 1}), + }, outcomes + + def test_a_model_endpoint_that_refuses_the_key_fails_the_task_as_an_infra_error(state, endpoint): endpoint.respond = answer("pong") diff --git a/tst/integration/cli/run_bundle_pydantic_ai_test.py b/tst/integration/cli/run_bundle_pydantic_ai_test.py index d57c62aa2..3829751e1 100644 --- a/tst/integration/cli/run_bundle_pydantic_ai_test.py +++ b/tst/integration/cli/run_bundle_pydantic_ai_test.py @@ -7,12 +7,15 @@ import logging import time import uuid +from collections import deque from typing import Final import httpx import pytest from click.testing import CliRunner +from agentenv_protocol.a2a_agent import Usage + from agent_env.a2a_agent import protocol from agent_env.artifact.store import reset_artifact_store from agent_env.cli import cli @@ -31,6 +34,16 @@ AGENT: Final[str] = "@local/agentenv-framework/agents/pydantic-ai" MODEL: Final[str] = "fake-model-1" ECHO_ENV: Final[str] = "@local/agentenv-framework/echo-mcp/echo" +# A tool call and an answer from Claude through a LiteLLM proxy, whose prompt leaves out the cache it wrote and read, +# and an answer from GPT through it, whose prompt counts the cache. +CLAUDE_CALLS: Final[list[dict]] = [ + {"prompt_tokens": 10, "completion_tokens": 129, "total_tokens": 139, "cache_creation_input_tokens": 10363, + "cache_read_input_tokens": 0}, + {"prompt_tokens": 13, "completion_tokens": 43, "total_tokens": 56, "cache_creation_input_tokens": 199, + "cache_read_input_tokens": 10363}, +] +GPT_CALL: Final[dict] = {"prompt_tokens": 2303, "completion_tokens": 2, "total_tokens": 2305, + "prompt_tokens_details": {"cached_tokens": 2048}} @pytest.fixture @@ -202,6 +215,35 @@ async def steps(context): assert {call.headers.get("x-agentenv-test") for call in endpoint.completions()} == {"header-check"} +def test_a_turns_usage_counts_the_cache_in_its_input_and_is_unknown_where_no_usage_came_back(state, endpoint): + outcomes = {} + endpoint.respond = answer("pong") + + async def steps(context): + base_url, _ = await _deployed(context) + claude = deque(CLAUDE_CALLS) + endpoint.respond = tool_then_answer("shell", {"command": "echo hi"}, prefix="the shell said: ") + endpoint.usage = lambda body: claude.popleft() if claude else None + outcomes["claude"] = await _send(base_url, "ctx", "Say hi with the shell.") + endpoint.respond, endpoint.usage = answer("pong"), lambda body: GPT_CALL + outcomes["gpt"] = await _send(base_url, "ctx", "Say pong.") + endpoint.usage = lambda body: None + outcomes["none"] = await _send(base_url, "ctx", "Say pong again.") + + _with_agent(state, steps) + + assert {outcome["state"] for outcome in outcomes.values()} == {"completed"}, outcomes + usage = {name: Usage.from_dict(outcome["data"]["usage"]) for name, outcome in outcomes.items()} + assert usage == { + "claude": Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, + provider_details={"requests": 2}), + "gpt": Usage(tool_call_count=0, input_tokens=2303, output_tokens=2, total_tokens=2305, cache_read_tokens=2048, + cache_write_tokens=0, input_includes_cache=True, provider_details={"requests": 1}), + "none": Usage(tool_call_count=0, provider_details={"requests": 1, "unreported_calls": 1}), + }, outcomes + + def test_a_model_endpoint_that_refuses_the_key_fails_the_task_as_an_infra_error(state, endpoint): endpoint.respond = answer("pong") diff --git a/tst/unit/agents/browser_use/test_browser_use_agent.py b/tst/unit/agents/browser_use/test_browser_use_agent.py index 881f277b5..848f2df85 100644 --- a/tst/unit/agents/browser_use/test_browser_use_agent.py +++ b/tst/unit/agents/browser_use/test_browser_use_agent.py @@ -58,7 +58,8 @@ def test_a_turn_returns_the_answer_and_a_context_keeps_one_conversation(client: assert (task["status"]["state"], text) == ("completed", "The top story.") usage = data["usage"] assert (usage["tool_call_count"], usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (2, 100, 20, 120) - assert usage["provider_details"]["cache_read_tokens"] == 60 and usage["provider_details"]["requests"] == 3 + assert (usage["cache_read_tokens"], usage["cache_write_tokens"], usage["input_includes_cache"]) == (60, 10, True) + assert usage["provider_details"]["requests"] == 3 and "cost_usd" not in usage turn = json.loads(record.read_text()) assert (turn["prompt"], turn["model"], turn["effort"], turn["systemPrompt"]) == ("top story?", "gpt-test", "low", "Be brief.") # Unset, browser-use's own step budget and mode; its run ends before the turn's deadline, leaving it to deliver. diff --git a/tst/unit/agents/hermes/test_message_contract.py b/tst/unit/agents/hermes/test_message_contract.py index beb946a3b..1b5986225 100644 --- a/tst/unit/agents/hermes/test_message_contract.py +++ b/tst/unit/agents/hermes/test_message_contract.py @@ -240,3 +240,25 @@ async def download(source, destination): assert not (target / "config.yaml").exists() assert (folders / "result.txt").read_text() == "42" assert turns.calls[-1]["session_id"] == "s-ctx-src" + + +def test_a_turns_usage_adds_the_cache_back_to_the_input(): + """GPT's two calls of a smoke run through a LiteLLM proxy, as Hermes reported them: its input leaves out the 10,496 + cached tokens the second call read.""" + report = {"input_tokens": 10750, "output_tokens": 48, "total_tokens": 21294, "cache_read_tokens": 10496, + "cache_write_tokens": 0, "reasoning_tokens": 0, "api_calls": 2, "estimated_cost_usd": 0.0, + "cost_status": "unknown"} + + assert a2a_server._usage(result(usage=report, tool_call_count=1)).to_dict() == { + "tool_call_count": 1, "input_tokens": 21246, "output_tokens": 48, "total_tokens": 21294, + "cache_read_tokens": 10496, "cache_write_tokens": 0, "input_includes_cache": True, "cost_usd": 0.0, + "provider_details": {"api_calls": 2, "reasoning_tokens": 0, "cost_status": "unknown"}} + + +def test_a_turn_whose_calls_reported_no_usage_has_unknown_counts(): + """Claude's streamed answers through a LiteLLM proxy carry no usage, so Hermes reports zeros for them.""" + report = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cache_read_tokens": 0, + "cache_write_tokens": 0, "api_calls": 3} + + assert a2a_server._usage(result(usage=report, tool_call_count=2)).to_dict() == { + "tool_call_count": 2, "provider_details": {"api_calls": 3, "unreported_calls": 3}} diff --git a/tst/unit/agents/openhands/test_openhands_agent.py b/tst/unit/agents/openhands/test_openhands_agent.py index aa911af7d..a686a44a0 100644 --- a/tst/unit/agents/openhands/test_openhands_agent.py +++ b/tst/unit/agents/openhands/test_openhands_agent.py @@ -9,6 +9,7 @@ from collections import Counter from dataclasses import dataclass, field from pathlib import Path +from types import SimpleNamespace from typing import Any, Final import httpx @@ -676,3 +677,37 @@ def test_the_card_version_is_the_pinned_sdk() -> None: assert f"openhands-sdk=={openhands_agent.OPENHANDS_VERSION}" in requirements assert f"openhands-tools=={openhands_agent.OPENHANDS_VERSION}" in requirements + + +def _call(prompt: int, completion: int, read: int = 0, write: int = 0) -> SimpleNamespace: + """The ``TokenUsage`` OpenHands keeps for a model call.""" + return SimpleNamespace(prompt_tokens=prompt, completion_tokens=completion, cache_read_tokens=read, + cache_write_tokens=write, reasoning_tokens=0) + + +def test_a_turns_usage_counts_the_cache_in_its_input() -> None: + """A smoke run's two calls through a LiteLLM proxy: Claude's through Chat Completions, whose prompt leaves the cache + out, and GPT's through the Responses API, whose input counts it.""" + claude = openhands_agent.turn_usage([(_call(10, 129, write=10363), False), (_call(13, 43, 10363, 199), False)], + responses=2, tool_calls=1, cost=0.0151) + gpt = openhands_agent.turn_usage([(_call(7803, 22), True), (_call(7878, 3, read=7680), True)], responses=2, + tool_calls=1, cost=0.004) + + assert claude == Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, cost_usd=0.0151, + provider_details={"reasoning_tokens": 0}) + assert gpt == Usage(tool_call_count=1, input_tokens=15681, output_tokens=25, total_tokens=15706, + cache_read_tokens=7680, cache_write_tokens=0, input_includes_cache=True, cost_usd=0.004, + provider_details={"reasoning_tokens": 0}) + + +def test_a_turns_usage_is_unknown_where_no_usage_came_back() -> None: + """A response that carried no usage, which OpenHands keeps no ``TokenUsage`` for, is counted apart, and a Chat + Completions prompt as large as its cache may or may not have counted it.""" + unreported = openhands_agent.turn_usage([], responses=1, tool_calls=0, cost=None) + ambiguous = openhands_agent.turn_usage([(_call(5000, 10, read=4000), False)], responses=2, tool_calls=2, cost=None) + + assert unreported == Usage(tool_call_count=0, provider_details={"reasoning_tokens": 0, "unreported_calls": 1}) + assert ambiguous == Usage(tool_call_count=2, input_tokens=5000, output_tokens=10, total_tokens=5010, + cache_read_tokens=4000, cache_write_tokens=0, input_includes_cache=None, + provider_details={"reasoning_tokens": 0, "unreported_calls": 1}) diff --git a/tst/unit/agents/pi/test_pi_agent.py b/tst/unit/agents/pi/test_pi_agent.py index 641c5bd3a..2ae8b6820 100644 --- a/tst/unit/agents/pi/test_pi_agent.py +++ b/tst/unit/agents/pi/test_pi_agent.py @@ -69,8 +69,8 @@ def test_run_returns_final_text_usage_and_trajectory(client: TestClient, record: assert task["status"]["state"] == "completed" assert text == "Done." assert data["usage"] == { - "tool_call_count": 1, "input_tokens": 200, "output_tokens": 40, "total_tokens": 252, "cost_usd": 0.5, - "provider_details": {"cache_read_tokens": 10, "cache_write_tokens": 2}, + "tool_call_count": 1, "input_tokens": 212, "output_tokens": 40, "total_tokens": 252, "cache_read_tokens": 10, + "cache_write_tokens": 2, "input_includes_cache": True, "cost_usd": 0.5, } invocation = json.loads(record.read_text()) assert invocation["stdin"] == 'fix the bug\n\n{\n "k": 1\n}' @@ -92,6 +92,34 @@ def test_run_returns_final_text_usage_and_trajectory(client: TestClient, record: assert "message_update" not in kinds +def _message(input: int, output: int, read: int, total: int) -> dict[str, Any]: + return {"role": "assistant", "usage": {"input": input, "output": output, "cacheRead": read, "cacheWrite": 0, + "totalTokens": total, "cost": {"total": 0}}} + + +def test_usage_adds_the_cache_back_to_the_input() -> None: + """GPT's two calls of a smoke run through a LiteLLM proxy, as pi recorded them: the second read 1,024 cached tokens + of its 1,139-token prompt, which pi's ``input`` leaves out.""" + run = pi_agent._Run(assistant_messages=[_message(1109, 21, 0, 1130), _message(115, 2, 1024, 1141)], tool_calls=1) + + assert pi_agent._usage(run).to_dict() == { + "tool_call_count": 1, "input_tokens": 2248, "output_tokens": 23, "total_tokens": 2271, + "cache_read_tokens": 1024, "cache_write_tokens": 0, "input_includes_cache": True, + } + + +def test_a_call_that_reported_no_usage_is_unknown_not_zero() -> None: + """Claude's streamed answers through a LiteLLM proxy carry no usage, so pi records zeros for them.""" + unreported = pi_agent._Run(assistant_messages=[_message(0, 0, 0, 0), _message(0, 0, 0, 0)], tool_calls=1) + partly = pi_agent._Run(assistant_messages=[_message(0, 0, 0, 0), _message(115, 2, 1024, 1141)]) + + assert pi_agent._usage(unreported).to_dict() == { + "tool_call_count": 1, "provider_details": {"unreported_calls": 2}} + assert pi_agent._usage(partly).to_dict() == { + "tool_call_count": 0, "input_tokens": 1139, "output_tokens": 2, "total_tokens": 1141, "cache_read_tokens": 1024, + "cache_write_tokens": 0, "input_includes_cache": True, "provider_details": {"unreported_calls": 1}} + + def test_a_context_keeps_one_pi_session(client: TestClient, record: Path) -> None: _send(client, [{"kind": "text", "text": "one"}], context_id="ctx/a b") first = json.loads(record.read_text())["argv"] diff --git a/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py b/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py index 39313b428..5d90797e6 100644 --- a/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py +++ b/tst/unit/agents/pydantic_ai/test_pydantic_ai_agent.py @@ -13,6 +13,7 @@ from collections import Counter from dataclasses import dataclass, field from pathlib import Path +from types import SimpleNamespace from typing import Any, Final import httpx @@ -697,3 +698,38 @@ async def cancel_midway() -> bool: assert asyncio.run(cancel_midway()) assert finished.is_set() + + +def _response(input: int, output: int, read: int = 0, write: int = 0, **details: int) -> SimpleNamespace: + """A ``ModelResponse``'s ``RequestUsage``: its cache counts, and the usage keys it doesn't read in ``details``.""" + return SimpleNamespace(input_tokens=input, output_tokens=output, cache_read_tokens=read, cache_write_tokens=write, + details=details) + + +def test_a_turns_usage_counts_the_cache_in_its_input() -> None: + """GPT's two calls of a smoke run through a LiteLLM proxy, whose prompt counts the cache, and Claude's, whose prompt + a LiteLLM proxy sends without it, with Anthropic's keys beside it, which Pydantic AI keeps in ``details``.""" + gpt = pydantic_ai_agent.turn_usage([_response(2169, 20), _response(2303, 2, read=2048)], tool_calls=1, requests=2) + claude = pydantic_ai_agent.turn_usage( + [_response(10, 129, cache_creation_input_tokens=10363, cache_read_input_tokens=0), + _response(13, 43, cache_creation_input_tokens=199, cache_read_input_tokens=10363)], tool_calls=1, requests=2) + + assert gpt == Usage(tool_call_count=1, input_tokens=4472, output_tokens=22, total_tokens=4494, + cache_read_tokens=2048, cache_write_tokens=0, input_includes_cache=True, + provider_details={"requests": 2}) + assert claude == Usage(tool_call_count=1, input_tokens=20948, output_tokens=172, total_tokens=21120, + cache_read_tokens=10363, cache_write_tokens=10562, input_includes_cache=True, + provider_details={"requests": 2}) + + +def test_a_turns_usage_is_unknown_where_no_usage_came_back() -> None: + """Claude's streamed answers through a LiteLLM proxy carry no usage, which Pydantic AI records as zeros; and a + prompt as large as the cache beside it may or may not have counted it.""" + unreported = pydantic_ai_agent.turn_usage([_response(0, 0), _response(0, 0)], tool_calls=1, requests=2) + ambiguous = pydantic_ai_agent.turn_usage( + [_response(0, 0), _response(5000, 10, cache_read_input_tokens=4000)], tool_calls=0, requests=2) + + assert unreported == Usage(tool_call_count=1, provider_details={"requests": 2, "unreported_calls": 2}) + assert ambiguous == Usage(tool_call_count=0, input_tokens=5000, output_tokens=10, total_tokens=5010, + cache_read_tokens=4000, cache_write_tokens=0, input_includes_cache=None, + provider_details={"requests": 2, "unreported_calls": 1}) diff --git a/tst/util/fake_openai.py b/tst/util/fake_openai.py index 4f47fbc6a..00b012ac1 100644 --- a/tst/util/fake_openai.py +++ b/tst/util/fake_openai.py @@ -1,7 +1,8 @@ """A fake OpenAI Chat Completions endpoint, so an agent that calls the model through LiteLLM runs with no model key: ``POST .../chat/completions`` is answered by a responder, as one JSON completion or, to a request that asks for a -stream, as server-sent chunks, and every request is recorded. A fault queued with ``fail_next`` replaces the next -answer: an HTTP error, or an answer that stalls.""" +stream, as server-sent chunks, and every request is recorded. Each answer carries the usage ``usage`` gives for its +request, none when it gives None. A fault queued with ``fail_next`` replaces the next answer: an HTTP error, or an +answer that stalls.""" import json import threading @@ -81,6 +82,7 @@ def tool_result_text(body: dict) -> str | None: class FakeOpenAI: host: str = "127.0.0.1" respond: Responder = field(default_factory=lambda: answer("ok")) + usage: Callable[[dict], dict | None] = lambda body: USAGE seen: list[Seen] = field(default_factory=list) _faults: deque = field(default_factory=deque) _server: ThreadingHTTPServer | None = None @@ -118,7 +120,7 @@ def do_POST(self) -> None: return if fault is not None and fault.kind == "stall": time.sleep(fault.seconds) - completion = _completion(body, fake.respond(body)) + completion = _completion(body, fake.respond(body), fake.usage(body)) if body.get("stream"): self._stream(completion) else: @@ -157,7 +159,8 @@ def __exit__(self, *exc) -> None: def _chunks(completion: dict) -> list[dict]: - """``completion`` as the chunks a stream sends: its message whole as one delta, its finish, then its usage.""" + """``completion`` as the chunks a stream sends: its message whole as one delta, its finish, then its usage if it + has one.""" (choice,) = completion["choices"] message = choice["message"] delta = {"role": "assistant", "content": message["content"]} @@ -167,11 +170,11 @@ def _chunks(completion: dict) -> list[dict]: return [ {**head, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}, {**head, "choices": [{"index": 0, "delta": {}, "finish_reason": choice["finish_reason"]}]}, - {**head, "choices": [], "usage": completion["usage"]}, + *([{**head, "choices": [], "usage": completion["usage"]}] if "usage" in completion else []), ] -def _completion(body: dict, reply: Text | ToolCall) -> dict: +def _completion(body: dict, reply: Text | ToolCall, usage: dict | None) -> dict: if isinstance(reply, Text): message, finish = {"role": "assistant", "content": reply.text}, "stop" else: @@ -181,4 +184,4 @@ def _completion(body: dict, reply: Text | ToolCall) -> dict: finish = "tool_calls" return {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}", "object": "chat.completion", "created": int(time.time()), "model": body.get("model", ""), "choices": [{"index": 0, "message": message, "finish_reason": finish}], - "usage": USAGE} + **({"usage": usage} if usage is not None else {})}