From 00fc4b622852785d18657d6d451755977f4e3fb7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:02:13 -0800 Subject: [PATCH 1/7] feat(protocol): Usage gains prompt-cache counts and an includes-cache flag, and the proxy reports its turns' model usage Usage takes cache_read_tokens, cache_write_tokens and input_includes_cache, validated like its other fields, and Usage.from_dict reads what any protocol version wrote. TaskResult.failure carries usage. The proxy sums the usage of the model calls its driver records into each turn's Usage, OpenAI- and Anthropic-shaped alike, with input_tokens counting the cache. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 18 ++++- .../src/agentenv_protocol/a2a_agent/proxy.py | 43 +++++++++++- .../agentenv_protocol/a2a_agent/tasks/v1.py | 58 +++++++++++----- .../agentenv-protocol/tests/test_a2a_proxy.py | 44 +++++++++++++ .../agentenv-protocol/tests/test_usage.py | 66 +++++++++++++++++++ src/agent_env/agents/AGENTS.md | 4 ++ .../agent/proxy_usage_local_test.py | 63 ++++++++++++++++++ tst/unit/a2a_agent/protocol_test.py | 23 ++++++- 8 files changed, 299 insertions(+), 20 deletions(-) create mode 100644 packages/agentenv-protocol/tests/test_usage.py create mode 100644 tst/integration/agent/proxy_usage_local_test.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index ec8b116e5..78a00c6f7 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1388,6 +1388,17 @@ Extensions come from explicit definitions, configured activations, and decorated handlers in one validated registry, so routes and card advertisement cannot drift. +`Usage` reports what a task used: `tool_call_count`, `input_tokens`, +`output_tokens`, `total_tokens`, `cost_usd`, the prompt-cache counts +`cache_read_tokens` and `cache_write_tokens`, and `input_includes_cache`, which +says whether `input_tokens` counts both cache counts (`True`), neither +(`False`), or is unknown (`None`, the default). Set the flag whenever you set +the cache counts: a price can't read them without it. Runtime-specific counts go +in `provider_details`, but cache counts go in their own fields. `Usage.from_dict` +reads what `to_dict` wrote under any protocol version, a field it leaves out +being `None` and a key it doesn't know ignored, and `TaskResult.failure(..., +usage=...)` carries usage on a failure as `success` does. + `run(request)` is the required execution contract. It is an ordinary method and needs no decorator. @@ -1830,7 +1841,12 @@ operations: records a call, each action must be one the last recorded call made, by name and arguments (a `bash` call by its command alone, a whole float equal to its int), and all of them must run before the next `record` or the `reply`, else 409. - Calls to tools the turn doesn't offer aren't checked. + Calls to tools the turn doesn't offer aren't checked. `usage` is the call's, in + OpenAI's shape (`prompt_tokens`, which counts `prompt_tokens_details.cached_tokens` + or LiteLLM's `cache_read_input_tokens` and `cache_creation_input_tokens`) or + Anthropic's (`input_tokens`, which counts neither). The turn's `Usage`, on its reply + or its error, sums each count over the calls that gave it, with `input_tokens` + counting the cache (`input_includes_cache` true); a value that isn't a count is left out. - `reply` (`{"token", "task_id", "parts"}`, or `"error"` in place of `"parts"`) ends the turn: parts complete the task, and an error fails it as `proxy.driver_error`. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py index 9c520d30c..d5b02c972 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py @@ -111,6 +111,7 @@ class _Turn: pending: list[tuple[str, str]] = field(default_factory=list) # (tool, its canonical arguments) not yet run last_event: str = field(default_factory=lambda: _now()) model: str | None = None + model_usage: list[Usage] = field(default_factory=list) # each recorded model call's @contextlib.asynccontextmanager @@ -175,11 +176,11 @@ async def run(self, request: TaskRequest[ProxyConfig]) -> TaskResult: if reply.error is not None: turn.trajectory.append(_span("turn", "chain", turn.started, turn.model, prompt=prompt, completion={"error": reply.error}, failed=True)) - return TaskResult.failure("proxy.driver_error", reply.error) + return TaskResult.failure("proxy.driver_error", reply.error, usage=_turn_usage(turn)) turn.trajectory.append(_span("turn", "chain", turn.started, turn.model, prompt=prompt, completion={ "content": [{"type": "text", "text": part.text} for part in reply.parts]})) return TaskResult.success(parts=tuple(TextPart(text=part.text) for part in reply.parts), - usage=Usage(tool_call_count=len(turn.actions))) + usage=_turn_usage(turn)) @extension(PROXY_V1.claim) async def claim(self, request: ProxyClaimRequest) -> None: @@ -249,6 +250,7 @@ async def record(self, request: ProxyRecordRequest) -> None: turn.model = request.model or turn.model span = _span("chat", "chat", turn.last_event, turn.model, completion={"content": content}) usage = request.usage or {} + turn.model_usage.append(_call_usage(usage)) for key, names in (("input_tokens", ("input_tokens", "prompt_tokens")), ("output_tokens", ("output_tokens", "completion_tokens"))): if (count := next((usage[name] for name in names if name in usage), None)) is not None: @@ -520,6 +522,43 @@ def _tool_error(message: str) -> ProxyToolResult: return ProxyToolResult(content=[{"type": "text", "text": f"error: {message}"}], is_error=True) +_USAGE_SUMS = ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", "cache_write_tokens") + + +def _call_usage(usage: dict[str, Any]) -> Usage: + """A recorded model call's usage, read from the shape its response carried it in, with ``input_tokens`` counting + the cache: OpenAI's ``prompt_tokens`` already counts it (``prompt_tokens_details.cached_tokens``, or the + ``cache_read_input_tokens`` and ``cache_creation_input_tokens`` LiteLLM sets beside it), and Anthropic's + ``input_tokens`` counts neither of those, so they're added. A value that isn't a count is left out.""" + details = usage.get("prompt_tokens_details") + cached = _count(details.get("cached_tokens")) if isinstance(details, dict) else None + read = cached if cached is not None else _count(usage.get("cache_read_input_tokens")) + write = _count(usage.get("cache_creation_input_tokens")) + if "prompt_tokens" in usage: + input_tokens = _count(usage["prompt_tokens"]) + elif (input_tokens := _count(usage.get("input_tokens"))) is not None: + input_tokens += (read or 0) + (write or 0) + output = _count(usage.get("completion_tokens", usage.get("output_tokens"))) + total = _count(usage.get("total_tokens")) + if total is None and input_tokens is not None and output is not None: + total = input_tokens + output + return Usage(input_tokens=input_tokens, output_tokens=output, total_tokens=total, cache_read_tokens=read, + cache_write_tokens=write, input_includes_cache=True if input_tokens is not None else None) + + +def _turn_usage(turn: _Turn) -> Usage: + """The turn's usage: the calls to its tools, and each count summed over the recorded model calls that gave it, + None when none did.""" + sums = {name: sum(values) if (values := [v for c in turn.model_usage if (v := getattr(c, name)) is not None]) + else None for name in _USAGE_SUMS} + return Usage(tool_call_count=len(turn.actions), **sums, + input_includes_cache=True if sums["input_tokens"] is not None else None) + + +def _count(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + def _span(name: str, operation: str, started: str, model: str | None, *, prompt: Any = None, completion: Any = None, failed: bool = False) -> dict[str, Any]: """An OTel GenAI span, the trajectory format rubrics and the explorer read.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 920ec7226..936f5e826 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -302,29 +302,44 @@ def __post_init__(self) -> None: object.__setattr__(self, "payload", _copy_json_value(self.payload)) +_USAGE_COUNTERS = ( + "tool_call_count", + "input_tokens", + "output_tokens", + "total_tokens", + "cache_read_tokens", + "cache_write_tokens", +) + + @dataclass(frozen=True, slots=True, kw_only=True) class Usage: - """Common execution metrics plus runtime-specific, JSON-shaped details.""" + """Common execution metrics plus runtime-specific, JSON-shaped details. ``cache_read_tokens`` and + ``cache_write_tokens`` count the prompt tokens read from and written to the provider's prompt cache; + ``input_includes_cache`` says whether ``input_tokens`` counts both of them (True), neither (False), or is unknown + (None), which a price needs to read the counts.""" tool_call_count: int | None = None input_tokens: int | None = None output_tokens: int | None = None total_tokens: int | None = None + cache_read_tokens: int | None = None + cache_write_tokens: int | None = None + input_includes_cache: bool | None = None cost_usd: float | None = None provider_details: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: - for name in ( - "tool_call_count", - "input_tokens", - "output_tokens", - "total_tokens", - ): + for name in _USAGE_COUNTERS: value = getattr(self, name) if value is not None and ( isinstance(value, bool) or not isinstance(value, int) or value < 0 ): - raise ValueError(f"{name} cannot be negative") + raise ValueError(f"{name} must be a non-negative integer") + if self.input_includes_cache is not None and not isinstance( + self.input_includes_cache, bool + ): + raise ValueError("input_includes_cache must be true, false or null") if self.cost_usd is not None and ( isinstance(self.cost_usd, bool) or not isinstance(self.cost_usd, (int, float)) @@ -337,23 +352,32 @@ def __post_init__(self) -> None: self, "provider_details", _copy_json_value(self.provider_details) ) + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "Usage": + """The usage ``to_dict`` wrote, from this protocol version or another: a field it leaves out is None, and a + key this version doesn't know is ignored. Raises ValueError or TypeError for a field it can't hold.""" + if not isinstance(data, Mapping): + raise TypeError("usage must be a mapping") + known = { + name: data[name] + for name in (*_USAGE_COUNTERS, "input_includes_cache", "cost_usd") + if data.get(name) is not None + } + return cls(**known, provider_details=data.get("provider_details") or {}) + @property def is_empty(self) -> bool: return ( - self.tool_call_count is None - and self.input_tokens is None - and self.output_tokens is None - and self.total_tokens is None + all(getattr(self, name) is None for name in _USAGE_COUNTERS) + and self.input_includes_cache is None and self.cost_usd is None and not self.provider_details ) def to_dict(self) -> dict[str, Any]: values = { - "tool_call_count": self.tool_call_count, - "input_tokens": self.input_tokens, - "output_tokens": self.output_tokens, - "total_tokens": self.total_tokens, + **{name: getattr(self, name) for name in _USAGE_COUNTERS}, + "input_includes_cache": self.input_includes_cache, "cost_usd": self.cost_usd, } result = {name: value for name, value in values.items() if value is not None} @@ -425,11 +449,13 @@ def failure( *, error_type: Literal["agent_error", "infra_error"] = "agent_error", parts: Iterable[TaskPart] | None = None, + usage: Usage | None = None, ) -> "TaskResult": return ( cls.builder() .failed(code, message, error_type=error_type) .parts(parts if parts is not None else (TextPart(text=message),)) + .usage(usage or Usage()) .build() ) diff --git a/packages/agentenv-protocol/tests/test_a2a_proxy.py b/packages/agentenv-protocol/tests/test_a2a_proxy.py index fb30d86ba..62da4fcd2 100644 --- a/packages/agentenv-protocol/tests/test_a2a_proxy.py +++ b/packages/agentenv-protocol/tests/test_a2a_proxy.py @@ -574,6 +574,50 @@ def test_recorded_model_calls_are_checked_against_the_actions_that_run(client, e assert task["status"]["message"]["parts"][1]["data"]["usage"]["tool_call_count"] == 3 +def _usage(task: dict[str, Any]) -> dict[str, Any]: + [usage] = [part["data"]["usage"] for part in task["status"]["message"]["parts"] + if part.get("kind") == "data" and "usage" in part["data"]] + return usage + + +def test_a_turns_usage_sums_its_recorded_calls_with_input_counting_the_cache(client): + task_id = _send(client, "go") + for usage in ({"prompt_tokens": 120, "completion_tokens": 7, "total_tokens": 127, + "prompt_tokens_details": {"cached_tokens": 100}}, + {"input_tokens": 3, "output_tokens": 9, "cache_read_input_tokens": 100, + "cache_creation_input_tokens": 20}, + {"prompt_tokens": 50, "completion_tokens": 1, "cache_read_input_tokens": 40, + "cache_creation_input_tokens": 10}, + None): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == { + "tool_call_count": 0, "input_tokens": 120 + 123 + 50, "output_tokens": 17, "total_tokens": 127 + 132 + 51, + "cache_read_tokens": 240, "cache_write_tokens": 30, "input_includes_cache": True} + + +def test_counts_a_recorded_call_doesnt_give_or_cant_be_read_are_left_out(client): + task_id = _send(client, "go") + for usage in ({"prompt_tokens": "9", "completion_tokens": True, "prompt_tokens_details": "none"}, + {"completion_tokens": 4}): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == {"tool_call_count": 0, "output_tokens": 4} + + +def test_an_error_reply_keeps_the_usage_recorded_before_it(client): + task_id = _send(client, "go") + _proxy(client, "record", task_id=task_id, message="Thinking.", usage={"input_tokens": 5, "output_tokens": 2}) + _proxy(client, "reply", task_id=task_id, error="the driver gave up") + + task = _settled(client, task_id) + assert task["status"]["state"] == "failed" + assert _usage(task) == {"tool_call_count": 0, "input_tokens": 5, "output_tokens": 2, "total_tokens": 7, + "input_includes_cache": True} + + def test_an_env_tool_named_bash_is_held_to_all_its_recorded_arguments(terminal, monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) with TestClient(ProxyAgent(shell=False, long_poll_seconds=0.2).create_app()) as client: diff --git a/packages/agentenv-protocol/tests/test_usage.py b/packages/agentenv-protocol/tests/test_usage.py new file mode 100644 index 000000000..922f5417a --- /dev/null +++ b/packages/agentenv-protocol/tests/test_usage.py @@ -0,0 +1,66 @@ +"""``Usage``'s cache counts and includes-cache flag: validated like its other fields, written by ``to_dict`` only when +set, read back by ``from_dict`` from any protocol version, and carried by a failed result as by a successful one.""" + +import pytest + +from agentenv_protocol.a2a_agent.tasks.v1 import TaskResult, Usage + +FULL = Usage(tool_call_count=2, input_tokens=120, output_tokens=7, total_tokens=127, cache_read_tokens=100, + cache_write_tokens=20, input_includes_cache=True, cost_usd=0.01, provider_details={"model": "m"}) + + +@pytest.mark.parametrize("usage", [FULL, Usage(), Usage(input_tokens=5), Usage(input_includes_cache=False), + Usage(cache_read_tokens=0, cache_write_tokens=0)], + ids=["full", "empty", "no-cache-fields", "flag-false", "zero-cache"]) +def test_usage_round_trips_with_and_without_the_cache_fields(usage): + assert Usage.from_dict(usage.to_dict()) == usage + + +def test_to_dict_writes_the_cache_fields_only_when_set(): + assert FULL.to_dict() == {"tool_call_count": 2, "input_tokens": 120, "output_tokens": 7, "total_tokens": 127, + "cache_read_tokens": 100, "cache_write_tokens": 20, "input_includes_cache": True, + "cost_usd": 0.01, "provider_details": {"model": "m"}} + assert Usage(input_tokens=5, input_includes_cache=False).to_dict() == {"input_tokens": 5, + "input_includes_cache": False} + + +def test_an_older_agents_usage_reads_with_no_cache_fields_and_a_newer_ones_unknown_keys_are_ignored(): + older = Usage.from_dict({"tool_call_count": 1, "input_tokens": 9, "output_tokens": 4, + "provider_details": {"cache_read_tokens": 3}}) + assert (older.cache_read_tokens, older.cache_write_tokens, older.input_includes_cache) == (None, None, None) + assert older.provider_details == {"cache_read_tokens": 3} + assert Usage.from_dict({"input_tokens": 9, "reasoning_tokens": 2, "cache_read_tokens": None}) == Usage( + input_tokens=9) + + +@pytest.mark.parametrize("field", ["cache_read_tokens", "cache_write_tokens"]) +@pytest.mark.parametrize("value", [-1, 1.5, "3", True]) +def test_a_cache_count_that_isnt_a_non_negative_integer_is_refused(field, value): + with pytest.raises(ValueError, match=f"{field} must be a non-negative integer"): + Usage(**{field: value}) + with pytest.raises(ValueError, match=field): + Usage.from_dict({field: value}) + + +@pytest.mark.parametrize("value", [1, 0, "true", "false"]) +def test_an_includes_cache_flag_that_isnt_a_bool_is_refused(value): + with pytest.raises(ValueError, match="input_includes_cache must be true, false or null"): + Usage(input_includes_cache=value) + + +def test_from_dict_refuses_what_isnt_a_mapping(): + with pytest.raises(TypeError, match="usage must be a mapping"): + Usage.from_dict([("input_tokens", 1)]) + + +@pytest.mark.parametrize("usage", [Usage(cache_read_tokens=0), Usage(cache_write_tokens=3), + Usage(input_includes_cache=False)], ids=["read", "write", "flag"]) +def test_a_usage_with_only_cache_fields_isnt_empty(usage): + assert not usage.is_empty + assert Usage().is_empty + + +def test_a_failed_result_carries_its_usage(): + result = TaskResult.failure("agent.boom", "it broke", usage=FULL) + assert (result.outcome, result.usage) == ("failed", FULL) + assert TaskResult.failure("agent.boom", "it broke").usage == Usage() diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index 0c1c0f558..db7645380 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -52,6 +52,10 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn. Its parts become the task's status message, with `usage` as a data part; a failure adds `{error_type, error_code, error_message}`. A success needs at least one part, and `builder().failed(...)` adds no text part, so add one. +- **Usage:** when the runtime counts prompt-cache tokens, put them in `Usage`'s `cache_read_tokens` and + `cache_write_tokens`, not `provider_details`, and set `input_includes_cache` to whether `input_tokens` counts them + (`None` when it can't tell). Code that sets them needs a `requirements.txt` protocol floor at the first release that + has them. - **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text never sent. diff --git a/tst/integration/agent/proxy_usage_local_test.py b/tst/integration/agent/proxy_usage_local_test.py new file mode 100644 index 000000000..d910feea0 --- /dev/null +++ b/tst/integration/agent/proxy_usage_local_test.py @@ -0,0 +1,63 @@ +"""A played agent's turn reaches the host with the usage of the model calls its trainer recorded: an episode's proxy, +served in this process, sums an OpenAI-shaped and an Anthropic-shaped call into one ``Usage`` whose ``input_tokens`` +counts the cache, and the host stores it with the agent's reply in the run's conversation. Local stores, no Docker.""" + +import pytest +from agentenv_protocol.a2a_agent.tasks.v1 import Usage + +from agent_env.a2a_agent import conversation_store +from agent_env.artifact.store import reset_artifact_store +from agent_env.config import configure, reset_config +from agent_env.task import EpisodeSpec, Task + +pytestmark = [pytest.mark.integration] + +OPENAI = {"prompt_tokens": 120, "completion_tokens": 7, "total_tokens": 127, "prompt_tokens_details": {"cached_tokens": 100}} +ANTHROPIC = {"input_tokens": 3, "output_tokens": 9, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20} + + +@pytest.fixture +def local_backends(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(tmp_path / "sandboxes")) + configure() + reset_artifact_store() + try: + yield + finally: + reset_config() + reset_artifact_store() + + +def _stored_usage(context_id: str) -> list[dict]: + """The usage of each agent message the run's conversation holds.""" + conversation = conversation_store.get_conversation(context_id) + return [part["data"]["usage"] for message in conversation["messages"] if message["role"] == "agent" + for part in message["parts"] if part.get("kind") == "data" and "usage" in (part.get("data") or {})] + + +@pytest.mark.asyncio +async def test_a_turns_recorded_model_calls_reach_the_host_as_the_turns_usage(local_backends): + task = Task.from_dict({"id": "usage", "steps": [ + {"id": "policy", "type": "deploy_agent", "agent_name": "policy", "env_ids": []}, + {"id": "play", "type": "prompt_agent", "agent_name": "policy", "prompt_id": "play", "prompt": "Say hi.", + "timeout_seconds": 60}, + ]}) + episode = await EpisodeSpec.from_task(task, trainer_agents=["policy"]).start() + turn = await episode.next_turn() + await turn.record_model_call({"role": "assistant", "content": "Thinking."}, model="policy-model", usage=OPENAI) + await turn.record_model_call("Hi.", model="policy-model", usage=ANTHROPIC) + await turn.end("Hi.") + await episode.result() + + record = Task.get_instance(episode.id) + assert record.status == "completed" + [play] = record.context["prompt_responses"] + assert play["response"] == "Hi." + [usage] = _stored_usage(play["a2a_context_id"]) + assert Usage.from_dict(usage) == Usage(tool_call_count=0, input_tokens=120 + 3 + 100 + 20, output_tokens=16, + total_tokens=127 + 123 + 9, cache_read_tokens=200, cache_write_tokens=20, + input_includes_cache=True) diff --git a/tst/unit/a2a_agent/protocol_test.py b/tst/unit/a2a_agent/protocol_test.py index 33548d8c5..e6d4f8e3b 100644 --- a/tst/unit/a2a_agent/protocol_test.py +++ b/tst/unit/a2a_agent/protocol_test.py @@ -22,7 +22,7 @@ raise_for_extension_status, wake_poll, ) -from agentenv_protocol.a2a_agent import TaskResult, Usage +from agentenv_protocol.a2a_agent import TaskResult, TextPart, Usage from agentenv_protocol.a2a_agent.framework import _to_a2a_parts @@ -135,6 +135,27 @@ def test_sdk_success_round_trips_through_platform_parser(): assert parsed.tool_call_count == 3 +def test_a_terminal_message_with_cache_usage_parses_and_its_usage_reads_back_whole(): + usage = Usage(tool_call_count=2, input_tokens=120, output_tokens=7, cache_read_tokens=100, cache_write_tokens=20, + input_includes_cache=True) + message = _sdk_terminal_message(TaskResult.success(parts=(TextPart(text="done"),), usage=usage)) + + parsed = TerminalResponse.from_message(message) + + assert (parsed.response_text, parsed.tool_call_count) == ("done", 2) + [sent] = [part["data"]["usage"] for part in message["parts"] if part.get("kind") == "data"] + assert Usage.from_dict(sent) == usage + + +def test_a_failed_turn_sends_its_usage(): + usage = Usage(input_tokens=5, output_tokens=2, input_includes_cache=False) + message = _sdk_terminal_message(TaskResult.failure("agent.boom", "it broke", usage=usage)) + + [sent] = [part["data"]["usage"] for part in message["parts"] if part.get("kind") == "data"] + assert Usage.from_dict(sent) == usage + assert TerminalResponse.from_message(message).error_code == "agent.boom" + + def test_sdk_failure_round_trips_through_platform_parser(): result = TaskResult.failure( "provider_rate_limited", From d30d9dd1bb9c6e9539949a750f1c954242db6bbb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:12:31 -0800 Subject: [PATCH 2/7] fix(protocol): read a LiteLLM proxy's prompt_tokens as leaving out the cache Anthropic's keys beside it name A real LiteLLM proxy answers Claude through /chat/completions with prompt_tokens counting neither cache count it sends beside it, while OpenAI and a LiteLLM client say what prompt_tokens counts through prompt_tokens_details.cached_tokens. The normalizer, now Usage.from_model_response so agents can read their own gateway calls the same way, adds the cache to prompt_tokens and total_tokens unless prompt_tokens_details says it's already there. The fixtures are the recorded responses. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 18 ++++++--- .../src/agentenv_protocol/a2a_agent/proxy.py | 27 +------------ .../agentenv_protocol/a2a_agent/tasks/v1.py | 32 +++++++++++++++ .../agentenv-protocol/tests/test_a2a_proxy.py | 2 +- .../agentenv-protocol/tests/test_usage.py | 39 +++++++++++++++++++ 5 files changed, 85 insertions(+), 33 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 78a00c6f7..e5241421e 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1399,6 +1399,15 @@ reads what `to_dict` wrote under any protocol version, a field it leaves out being `None` and a key it doesn't know ignored, and `TaskResult.failure(..., usage=...)` carries usage on a failure as `success` does. +`Usage.from_model_response(usage)` reads a model call's usage as an OpenAI- or +Anthropic-style response carries it, with `input_tokens` counting the cache +(`input_includes_cache` true). `prompt_tokens` counts the cache when +`prompt_tokens_details.cached_tokens` says what was cached, as OpenAI's and a +LiteLLM client's do. Anthropic's `input_tokens` counts neither +`cache_read_input_tokens` nor `cache_creation_input_tokens`, and neither does +the `prompt_tokens` a LiteLLM proxy sends with those two beside it, so they are +added, to `total_tokens` too. A value that isn't a count is left out. + `run(request)` is the required execution contract. It is an ordinary method and needs no decorator. @@ -1841,12 +1850,9 @@ operations: records a call, each action must be one the last recorded call made, by name and arguments (a `bash` call by its command alone, a whole float equal to its int), and all of them must run before the next `record` or the `reply`, else 409. - Calls to tools the turn doesn't offer aren't checked. `usage` is the call's, in - OpenAI's shape (`prompt_tokens`, which counts `prompt_tokens_details.cached_tokens` - or LiteLLM's `cache_read_input_tokens` and `cache_creation_input_tokens`) or - Anthropic's (`input_tokens`, which counts neither). The turn's `Usage`, on its reply - or its error, sums each count over the calls that gave it, with `input_tokens` - counting the cache (`input_includes_cache` true); a value that isn't a count is left out. + Calls to tools the turn doesn't offer aren't checked. `usage` is the call's, as the + model's response carried it, read by `Usage.from_model_response`. The turn's `Usage`, on + its reply or its error, sums each count over the calls that gave it. - `reply` (`{"token", "task_id", "parts"}`, or `"error"` in place of `"parts"`) ends the turn: parts complete the task, and an error fails it as `proxy.driver_error`. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py index d5b02c972..5e1abd131 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py @@ -250,7 +250,7 @@ async def record(self, request: ProxyRecordRequest) -> None: turn.model = request.model or turn.model span = _span("chat", "chat", turn.last_event, turn.model, completion={"content": content}) usage = request.usage or {} - turn.model_usage.append(_call_usage(usage)) + turn.model_usage.append(Usage.from_model_response(usage)) for key, names in (("input_tokens", ("input_tokens", "prompt_tokens")), ("output_tokens", ("output_tokens", "completion_tokens"))): if (count := next((usage[name] for name in names if name in usage), None)) is not None: @@ -525,27 +525,6 @@ def _tool_error(message: str) -> ProxyToolResult: _USAGE_SUMS = ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", "cache_write_tokens") -def _call_usage(usage: dict[str, Any]) -> Usage: - """A recorded model call's usage, read from the shape its response carried it in, with ``input_tokens`` counting - the cache: OpenAI's ``prompt_tokens`` already counts it (``prompt_tokens_details.cached_tokens``, or the - ``cache_read_input_tokens`` and ``cache_creation_input_tokens`` LiteLLM sets beside it), and Anthropic's - ``input_tokens`` counts neither of those, so they're added. A value that isn't a count is left out.""" - details = usage.get("prompt_tokens_details") - cached = _count(details.get("cached_tokens")) if isinstance(details, dict) else None - read = cached if cached is not None else _count(usage.get("cache_read_input_tokens")) - write = _count(usage.get("cache_creation_input_tokens")) - if "prompt_tokens" in usage: - input_tokens = _count(usage["prompt_tokens"]) - elif (input_tokens := _count(usage.get("input_tokens"))) is not None: - input_tokens += (read or 0) + (write or 0) - output = _count(usage.get("completion_tokens", usage.get("output_tokens"))) - total = _count(usage.get("total_tokens")) - if total is None and input_tokens is not None and output is not None: - total = input_tokens + output - return Usage(input_tokens=input_tokens, output_tokens=output, total_tokens=total, cache_read_tokens=read, - cache_write_tokens=write, input_includes_cache=True if input_tokens is not None else None) - - def _turn_usage(turn: _Turn) -> Usage: """The turn's usage: the calls to its tools, and each count summed over the recorded model calls that gave it, None when none did.""" @@ -555,10 +534,6 @@ def _turn_usage(turn: _Turn) -> Usage: input_includes_cache=True if sums["input_tokens"] is not None else None) -def _count(value: Any) -> int | None: - return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None - - def _span(name: str, operation: str, started: str, model: str | None, *, prompt: Any = None, completion: Any = None, failed: bool = False) -> dict[str, Any]: """An OTel GenAI span, the trajectory format rubrics and the explorer read.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 936f5e826..874216966 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -312,6 +312,10 @@ def __post_init__(self) -> None: ) +def _count(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + @dataclass(frozen=True, slots=True, kw_only=True) class Usage: """Common execution metrics plus runtime-specific, JSON-shaped details. ``cache_read_tokens`` and @@ -365,6 +369,34 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Usage": } return cls(**known, provider_details=data.get("provider_details") or {}) + @classmethod + def from_model_response(cls, usage: Mapping[str, Any]) -> "Usage": + """A model call's usage as its response carried it, OpenAI's way or Anthropic's, with ``input_tokens`` + counting the cache. OpenAI's ``prompt_tokens`` counts what ``prompt_tokens_details.cached_tokens`` says was + cached, and so does a LiteLLM client's, which also counts ``cache_creation_input_tokens``; Anthropic's + ``input_tokens``, and the ``prompt_tokens`` a LiteLLM proxy sends with Anthropic's + ``cache_read_input_tokens`` and ``cache_creation_input_tokens`` beside it, count neither, so they're added. A + value that isn't a count is left out.""" + details = usage.get("prompt_tokens_details") + cached = _count(details.get("cached_tokens")) if isinstance(details, Mapping) else None + read = cached if cached is not None else _count(usage.get("cache_read_input_tokens")) + write = _count(usage.get("cache_creation_input_tokens")) + cache = (read or 0) + (write or 0) + added = 0 # the cache the response's own counts leave out + if "prompt_tokens" in usage: + input_tokens = _count(usage["prompt_tokens"]) + if input_tokens is not None and (cached is None or input_tokens < cache): + input_tokens, added = input_tokens + cache, cache + elif (input_tokens := _count(usage.get("input_tokens"))) is not None: + input_tokens, added = input_tokens + cache, cache + output = _count(usage.get("completion_tokens", usage.get("output_tokens"))) + if (total := _count(usage.get("total_tokens"))) is not None: + total += added + elif input_tokens is not None and output is not None: + total = input_tokens + output + return cls(input_tokens=input_tokens, output_tokens=output, total_tokens=total, cache_read_tokens=read, + cache_write_tokens=write, input_includes_cache=True if input_tokens is not None else None) + @property def is_empty(self) -> bool: return ( diff --git a/packages/agentenv-protocol/tests/test_a2a_proxy.py b/packages/agentenv-protocol/tests/test_a2a_proxy.py index 62da4fcd2..6fb2fdaa5 100644 --- a/packages/agentenv-protocol/tests/test_a2a_proxy.py +++ b/packages/agentenv-protocol/tests/test_a2a_proxy.py @@ -593,7 +593,7 @@ def test_a_turns_usage_sums_its_recorded_calls_with_input_counting_the_cache(cli _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) assert _usage(_settled(client, task_id)) == { - "tool_call_count": 0, "input_tokens": 120 + 123 + 50, "output_tokens": 17, "total_tokens": 127 + 132 + 51, + "tool_call_count": 0, "input_tokens": 120 + 123 + 100, "output_tokens": 17, "total_tokens": 127 + 132 + 101, "cache_read_tokens": 240, "cache_write_tokens": 30, "input_includes_cache": True} diff --git a/packages/agentenv-protocol/tests/test_usage.py b/packages/agentenv-protocol/tests/test_usage.py index 922f5417a..42b326620 100644 --- a/packages/agentenv-protocol/tests/test_usage.py +++ b/packages/agentenv-protocol/tests/test_usage.py @@ -64,3 +64,42 @@ def test_a_failed_result_carries_its_usage(): result = TaskResult.failure("agent.boom", "it broke", usage=FULL) assert (result.outcome, result.usage) == ("failed", FULL) assert TaskResult.failure("agent.boom", "it broke").usage == Usage() + + +# Real responses to one Claude prompt of 7,818 tokens, 7,809 of them cacheable, and one GPT prompt, recorded through +# a LiteLLM proxy and, for the client's shape, as LiteLLM 1.83's client reads Anthropic's. +RESPONSES = { + "litellm-proxy-claude-writes": ({"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14, + "cache_creation_input_tokens": 7809, "cache_read_input_tokens": 0}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, cache_read_tokens=0, + cache_write_tokens=7809, input_includes_cache=True)), + "litellm-proxy-claude-reads": ({"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14, + "cache_creation_input_tokens": 0, "cache_read_input_tokens": 7809}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, cache_read_tokens=7809, + cache_write_tokens=0, input_includes_cache=True)), + "anthropic-messages-reads": ({"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7809, "cache_creation": {"ephemeral_5m_input_tokens": 0}}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, cache_read_tokens=7809, + cache_write_tokens=0, input_includes_cache=True)), + "litellm-client-claude-reads": ({"prompt_tokens": 7818, "completion_tokens": 5, "total_tokens": 7823, + "prompt_tokens_details": {"cached_tokens": 7809}, + "cache_creation_input_tokens": 0, "cache_read_input_tokens": 7809}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, + cache_read_tokens=7809, cache_write_tokens=0, input_includes_cache=True)), + "openai-gpt-reads": ({"prompt_tokens": 7822, "completion_tokens": 2, "total_tokens": 7824, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 7680}}, + Usage(input_tokens=7822, output_tokens=2, total_tokens=7824, cache_read_tokens=7680, + input_includes_cache=True)), + "openai-no-details": ({"prompt_tokens": 9, "completion_tokens": 4}, + Usage(input_tokens=9, output_tokens=4, total_tokens=13, input_includes_cache=True)), + "details-below-the-cache": ({"prompt_tokens": 9, "completion_tokens": 1, "prompt_tokens_details": {"cached_tokens": 50}}, + Usage(input_tokens=59, output_tokens=1, total_tokens=60, cache_read_tokens=50, + input_includes_cache=True)), + "nothing": ({}, Usage()), + "unreadable": ({"prompt_tokens": "9", "completion_tokens": True, "prompt_tokens_details": "x"}, Usage()), +} + + +@pytest.mark.parametrize("response, expected", RESPONSES.values(), ids=RESPONSES.keys()) +def test_a_model_responses_usage_reads_with_input_counting_the_cache(response, expected): + assert Usage.from_model_response(response) == expected From c41e05b51ad159a9b3aa58c63c3b5d94074210be Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 15:14:48 -0800 Subject: [PATCH 3/7] fix(protocol): Usage.from_dict refuses provider_details that aren't a mapping, empty ones too; the usage episode test abandons its episode Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/a2a_agent/tasks/v1.py | 3 ++- packages/agentenv-protocol/tests/test_usage.py | 7 +++++++ tst/integration/agent/proxy_usage_local_test.py | 14 +++++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 874216966..6092e9faf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -367,7 +367,8 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Usage": for name in (*_USAGE_COUNTERS, "input_includes_cache", "cost_usd") if data.get(name) is not None } - return cls(**known, provider_details=data.get("provider_details") or {}) + details = data.get("provider_details") + return cls(**known, provider_details={} if details is None else details) @classmethod def from_model_response(cls, usage: Mapping[str, Any]) -> "Usage": diff --git a/packages/agentenv-protocol/tests/test_usage.py b/packages/agentenv-protocol/tests/test_usage.py index 42b326620..eedd71378 100644 --- a/packages/agentenv-protocol/tests/test_usage.py +++ b/packages/agentenv-protocol/tests/test_usage.py @@ -53,6 +53,13 @@ def test_from_dict_refuses_what_isnt_a_mapping(): Usage.from_dict([("input_tokens", 1)]) +@pytest.mark.parametrize("details", [[], False, 0, "", "x", [("a", 1)]]) +def test_from_dict_refuses_provider_details_that_arent_a_mapping_even_when_empty(details): + with pytest.raises(TypeError, match="provider_details must be a mapping"): + Usage.from_dict({"input_tokens": 1, "provider_details": details}) + assert Usage.from_dict({"input_tokens": 1, "provider_details": None}) == Usage(input_tokens=1) + + @pytest.mark.parametrize("usage", [Usage(cache_read_tokens=0), Usage(cache_write_tokens=3), Usage(input_includes_cache=False)], ids=["read", "write", "flag"]) def test_a_usage_with_only_cache_fields_isnt_empty(usage): diff --git a/tst/integration/agent/proxy_usage_local_test.py b/tst/integration/agent/proxy_usage_local_test.py index d910feea0..c728e487b 100644 --- a/tst/integration/agent/proxy_usage_local_test.py +++ b/tst/integration/agent/proxy_usage_local_test.py @@ -47,11 +47,15 @@ async def test_a_turns_recorded_model_calls_reach_the_host_as_the_turns_usage(lo "timeout_seconds": 60}, ]}) episode = await EpisodeSpec.from_task(task, trainer_agents=["policy"]).start() - turn = await episode.next_turn() - await turn.record_model_call({"role": "assistant", "content": "Thinking."}, model="policy-model", usage=OPENAI) - await turn.record_model_call("Hi.", model="policy-model", usage=ANTHROPIC) - await turn.end("Hi.") - await episode.result() + try: + turn = await episode.next_turn() + await turn.record_model_call({"role": "assistant", "content": "Thinking."}, model="policy-model", + usage=OPENAI) + await turn.record_model_call("Hi.", model="policy-model", usage=ANTHROPIC) + await turn.end("Hi.") + await episode.result() + finally: + await episode.abandon("the test ended") record = Task.get_instance(episode.id) assert record.status == "completed" From 90bbcea5adcb31c93f1458dc95b8ed7f71180adf Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 16:14:25 -0800 Subject: [PATCH 4/7] fix(protocol): the proxy's chat spans count what its turn usage sums, and a sum short of some calls says so A chat span took its token counts from the raw usage while the turn's Usage read it with from_model_response, so a LiteLLM proxy's Claude call showed 9 input tokens in the trajectory and 7,818 in the usage. Both now read the call once. The turn's provider_details give model_calls and, for a count only some calls gave, how many left it out. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 7 +++-- .../src/agentenv_protocol/a2a_agent/proxy.py | 24 ++++++++++------ .../agentenv-protocol/tests/test_a2a_proxy.py | 28 +++++++++++++++++-- .../agent/proxy_usage_local_test.py | 3 +- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 6d200ab10..5f3f317ee 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1887,8 +1887,11 @@ operations: arguments (a `bash` call by its command alone, a whole float equal to its int), and all of them must run before the next `record` or the `reply`, else 409. Calls to tools the turn doesn't offer aren't checked. `usage` is the call's, as the - model's response carried it, read by `Usage.from_model_response`. The turn's `Usage`, on - its reply or its error, sums each count over the calls that gave it. + model's response carried it, read by `Usage.from_model_response`, and its `chat` span takes + its token counts from that reading. The turn's `Usage`, on its reply or its error, sums each + count over the calls that gave it; its `provider_details` give `model_calls`, how many were + recorded, and `unreported`, how many left out each count only some gave, so such a sum reads + as a lower bound. - `reply` (`{"token", "task_id", "parts"}`, or `"error"` in place of `"parts"`) ends the turn: parts complete the task, and an error fails it as `proxy.driver_error`. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py index 7019cb4cc..6bb368342 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py @@ -248,11 +248,10 @@ async def record(self, request: ProxyRecordRequest) -> None: turn.recorded = True turn.model = request.model or turn.model span = _span("chat", "chat", turn.last_event, turn.model, completion={"content": content}) - usage = request.usage or {} - turn.model_usage.append(Usage.from_model_response(usage)) - for key, names in (("input_tokens", ("input_tokens", "prompt_tokens")), - ("output_tokens", ("output_tokens", "completion_tokens"))): - if (count := next((usage[name] for name in names if name in usage), None)) is not None: + usage = Usage.from_model_response(request.usage or {}) + turn.model_usage.append(usage) + for key in ("input_tokens", "output_tokens"): + if (count := getattr(usage, key)) is not None: span["attributes"][f"gen_ai.usage.{key}"] = count turn.trajectory.append(span) turn.last_event = span["end_time"] @@ -526,11 +525,18 @@ def _tool_error(message: str) -> ProxyToolResult: def _turn_usage(turn: _Turn) -> Usage: """The turn's usage: the calls to its tools, and each count summed over the recorded model calls that gave it, - None when none did.""" - sums = {name: sum(values) if (values := [v for c in turn.model_usage if (v := getattr(c, name)) is not None]) - else None for name in _USAGE_SUMS} + None when none did. ``provider_details`` says how many calls were recorded and, for a count only some of them + gave, how many left it out, so a sum short of some calls reads as the lower bound it is.""" + sums, unreported = {}, {} + for name in _USAGE_SUMS: + values = [value for call in turn.model_usage if (value := getattr(call, name)) is not None] + sums[name] = sum(values) if values else None + if values and len(values) < len(turn.model_usage): + unreported[name] = len(turn.model_usage) - len(values) + details = {"model_calls": len(turn.model_usage), **({"unreported": unreported} if unreported else {})} return Usage(tool_call_count=len(turn.actions), **sums, - input_includes_cache=True if sums["input_tokens"] is not None else None) + input_includes_cache=True if sums["input_tokens"] is not None else None, + provider_details=details if turn.model_usage else {}) def _span(name: str, operation: str, started: str, model: str | None, *, prompt: Any = None, diff --git a/packages/agentenv-protocol/tests/test_a2a_proxy.py b/packages/agentenv-protocol/tests/test_a2a_proxy.py index 17251e4b1..e3bcb333f 100644 --- a/packages/agentenv-protocol/tests/test_a2a_proxy.py +++ b/packages/agentenv-protocol/tests/test_a2a_proxy.py @@ -598,7 +598,9 @@ def test_a_turns_usage_sums_its_recorded_calls_with_input_counting_the_cache(cli assert _usage(_settled(client, task_id)) == { "tool_call_count": 0, "input_tokens": 120 + 123 + 100, "output_tokens": 17, "total_tokens": 127 + 132 + 101, - "cache_read_tokens": 240, "cache_write_tokens": 30, "input_includes_cache": True} + "cache_read_tokens": 240, "cache_write_tokens": 30, "input_includes_cache": True, + "provider_details": {"model_calls": 4, "unreported": { + "input_tokens": 1, "output_tokens": 1, "total_tokens": 1, "cache_read_tokens": 1, "cache_write_tokens": 2}}} def test_counts_a_recorded_call_doesnt_give_or_cant_be_read_are_left_out(client): @@ -608,7 +610,8 @@ def test_counts_a_recorded_call_doesnt_give_or_cant_be_read_are_left_out(client) assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) - assert _usage(_settled(client, task_id)) == {"tool_call_count": 0, "output_tokens": 4} + assert _usage(_settled(client, task_id)) == {"tool_call_count": 0, "output_tokens": 4, "provider_details": { + "model_calls": 2, "unreported": {"output_tokens": 1}}} def test_an_error_reply_keeps_the_usage_recorded_before_it(client): @@ -619,7 +622,26 @@ def test_an_error_reply_keeps_the_usage_recorded_before_it(client): task = _settled(client, task_id) assert task["status"]["state"] == "failed" assert _usage(task) == {"tool_call_count": 0, "input_tokens": 5, "output_tokens": 2, "total_tokens": 7, - "input_includes_cache": True} + "input_includes_cache": True, "provider_details": {"model_calls": 1}} + + +def test_a_chat_spans_counts_are_the_ones_the_turns_usage_sums(client): + task_id = _send(client, "go") + _proxy(client, "record", task_id=task_id, message="Thinking.", usage={ + "prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14, "cache_read_input_tokens": 7809}) + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + usage = _usage(_settled(client, task_id)) + [chat, _] = _trajectory(client, task_id) + assert (chat["attributes"]["gen_ai.usage.input_tokens"], chat["attributes"]["gen_ai.usage.output_tokens"]) == ( + usage["input_tokens"], usage["output_tokens"]) == (7818, 5) + + +def test_a_turn_with_no_recorded_calls_reports_only_its_tool_calls(client): + task_id = _send(client, "go") + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == {"tool_call_count": 0} def test_an_env_tool_named_bash_is_held_to_all_its_recorded_arguments(terminal, monkeypatch, tmp_path): diff --git a/tst/integration/agent/proxy_usage_local_test.py b/tst/integration/agent/proxy_usage_local_test.py index c728e487b..0f52422b4 100644 --- a/tst/integration/agent/proxy_usage_local_test.py +++ b/tst/integration/agent/proxy_usage_local_test.py @@ -64,4 +64,5 @@ async def test_a_turns_recorded_model_calls_reach_the_host_as_the_turns_usage(lo [usage] = _stored_usage(play["a2a_context_id"]) assert Usage.from_dict(usage) == Usage(tool_call_count=0, input_tokens=120 + 3 + 100 + 20, output_tokens=16, total_tokens=127 + 123 + 9, cache_read_tokens=200, cache_write_tokens=20, - input_includes_cache=True) + input_includes_cache=True, + provider_details={"model_calls": 2, "unreported": {"cache_write_tokens": 1}}) From 6646ee64f78f46e88fd9fbfef2640a47d50ca41d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 16:28:14 -0800 Subject: [PATCH 5/7] refactor(protocol): Usage's field list is written once, and from_model_response decides once whether a response's input counts the cache from_dict, is_empty, to_dict and the proxy's sums read Usage's fields from the dataclass, so a count added to Usage reaches them all. from_model_response makes the one decision its docstring describes and applies it to the input and the total together. Behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/a2a_agent/proxy.py | 4 +- .../agentenv_protocol/a2a_agent/tasks/v1.py | 41 ++++++++----------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py index 6bb368342..e9e08a70d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py @@ -19,7 +19,7 @@ import signal from collections import defaultdict from collections.abc import AsyncIterator -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -520,7 +520,7 @@ def _tool_error(message: str) -> ProxyToolResult: return ProxyToolResult(content=[{"type": "text", "text": f"error: {message}"}], is_error=True) -_USAGE_SUMS = ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", "cache_write_tokens") +_USAGE_SUMS = tuple(f.name for f in fields(Usage) if f.name.endswith("_tokens")) def _turn_usage(turn: _Turn) -> Usage: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index e1975de77..54f2a1a6d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -11,7 +11,7 @@ import json import logging from collections.abc import AsyncIterator, Awaitable, Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from enum import Enum from pathlib import Path from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union @@ -376,9 +376,9 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Usage": if not isinstance(data, Mapping): raise TypeError("usage must be a mapping") known = { - name: data[name] - for name in (*_USAGE_COUNTERS, "input_includes_cache", "cost_usd") - if data.get(name) is not None + f.name: data[f.name] + for f in fields(cls) + if f.name != "provider_details" and data.get(f.name) is not None } details = data.get("provider_details") return cls(**known, provider_details={} if details is None else details) @@ -396,16 +396,15 @@ def from_model_response(cls, usage: Mapping[str, Any]) -> "Usage": read = cached if cached is not None else _count(usage.get("cache_read_input_tokens")) write = _count(usage.get("cache_creation_input_tokens")) cache = (read or 0) + (write or 0) - added = 0 # the cache the response's own counts leave out - if "prompt_tokens" in usage: - input_tokens = _count(usage["prompt_tokens"]) - if input_tokens is not None and (cached is None or input_tokens < cache): - input_tokens, added = input_tokens + cache, cache - elif (input_tokens := _count(usage.get("input_tokens"))) is not None: - input_tokens, added = input_tokens + cache, cache + openai = "prompt_tokens" in usage + input_tokens = _count(usage["prompt_tokens"] if openai else usage.get("input_tokens")) + counts_cache = openai and cached is not None and input_tokens is not None and input_tokens >= cache + missing = cache if input_tokens is not None and not counts_cache else 0 + if input_tokens is not None: + input_tokens += missing output = _count(usage.get("completion_tokens", usage.get("output_tokens"))) if (total := _count(usage.get("total_tokens"))) is not None: - total += added + total += missing elif input_tokens is not None and output is not None: total = input_tokens + output return cls(input_tokens=input_tokens, output_tokens=output, total_tokens=total, cache_read_tokens=read, @@ -413,24 +412,18 @@ def from_model_response(cls, usage: Mapping[str, Any]) -> "Usage": @property def is_empty(self) -> bool: - return ( - all(getattr(self, name) is None for name in _USAGE_COUNTERS) - and self.input_includes_cache is None - and self.cost_usd is None - and not self.provider_details - ) + return all(value is None for value in self._values().values()) and not self.provider_details def to_dict(self) -> dict[str, Any]: - values = { - **{name: getattr(self, name) for name in _USAGE_COUNTERS}, - "input_includes_cache": self.input_includes_cache, - "cost_usd": self.cost_usd, - } - result = {name: value for name, value in values.items() if value is not None} + result = {name: value for name, value in self._values().items() if value is not None} if self.provider_details: result["provider_details"] = _thaw(self.provider_details) return result + def _values(self) -> dict[str, Any]: + """Every field but ``provider_details``, by name, in declaration order.""" + return {f.name: getattr(self, f.name) for f in fields(self) if f.name != "provider_details"} + @dataclass(frozen=True, slots=True, kw_only=True) class TaskResult: From 71e23c1b6fce9a57dc9cccc61a9d96e22138024e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 20:23:28 -0800 Subject: [PATCH 6/7] feat(protocol): a driver can report a call's Usage itself, and the default reader knows the Responses shape Whoever makes a model call knows what its counts hold. record_model_call takes a Usage, sent to the proxy under REPORTED_USAGE_KEY inside the record's usage, so a proxy on an earlier protocol still takes the record, and taken as given; one that isn't a Usage gets 400 and leaves the turn as it was. Each call's input is made to count the cache before its span and its turn read it, a call whose flag is unknown while it has cache counts leaving the turn's flag unknown, and the turn sums cost_usd too. Usage.from_model_response, the default for a response's own usage, reads OpenAI's Responses shape and the cache_write_tokens OpenAI's details now give, keeps each provider's split of its cache writes by how long they're kept (priced apart) in provider_details, and marks a usage in no shape it knows as unrecognized. The fixtures are responses recorded through a LiteLLM proxy. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 47 +++++++++---- .../agentenv_protocol/a2a_agent/__init__.py | 2 + .../agentenv_protocol/a2a_agent/extensions.py | 7 ++ .../src/agentenv_protocol/a2a_agent/proxy.py | 66 ++++++++++++++---- .../agentenv_protocol/a2a_agent/tasks/v1.py | 46 +++++++++---- .../agentenv-protocol/tests/test_a2a_proxy.py | 68 ++++++++++++++++++- .../agentenv-protocol/tests/test_usage.py | 34 +++++++++- src/agent_env/agents/AGENTS.md | 5 +- src/agent_env/task/episode/run.py | 9 ++- .../agent/proxy_usage_local_test.py | 19 ++++-- 10 files changed, 249 insertions(+), 54 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 5f3f317ee..09e21129b 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1399,14 +1399,27 @@ reads what `to_dict` wrote under any protocol version, a field it leaves out being `None` and a key it doesn't know ignored, and `TaskResult.failure(..., usage=...)` carries usage on a failure as `success` does. -`Usage.from_model_response(usage)` reads a model call's usage as an OpenAI- or -Anthropic-style response carries it, with `input_tokens` counting the cache -(`input_includes_cache` true). `prompt_tokens` counts the cache when -`prompt_tokens_details.cached_tokens` says what was cached, as OpenAI's and a -LiteLLM client's do. Anthropic's `input_tokens` counts neither -`cache_read_input_tokens` nor `cache_creation_input_tokens`, and neither does -the `prompt_tokens` a LiteLLM proxy sends with those two beside it, so they are -added, to `total_tokens` too. A value that isn't a count is left out. +Whoever makes a model call knows what its counts hold, so an agent builds its own +`Usage`. `Usage.from_model_response(usage)` is the default reader for a model +response's own usage, with `input_tokens` counting the cache +(`input_includes_cache` true), for OpenAI's Chat Completions (`prompt_tokens`) +and Responses (`input_tokens`) shapes and Anthropic's: + +- An OpenAI count, or a LiteLLM client's, counts the cache its + `prompt_tokens_details` or `input_tokens_details` names: `cached_tokens`, and + `cache_write_tokens` beside it. +- Anthropic's `input_tokens` counts neither `cache_read_input_tokens` nor + `cache_creation_input_tokens`, and neither does the `prompt_tokens` a LiteLLM + proxy sends with those two beside it, so they're added, to `total_tokens` too. +- A provider's split of its cache writes by how long they're kept, priced apart + (Anthropic's `cache_creation`, OpenAI's `cache_write_tokens_details`), is kept + in `provider_details`, and so are the keys of a usage in none of these shapes, + as `unrecognized`. A value that isn't a count is left out. + +Through a gateway, whether a response's input counts the cache depends on the +model behind it: a LiteLLM proxy's Claude answers through Chat Completions leave +it out, its GPT answers don't. Set `input_includes_cache` per call, never once +per agent. `run(request)` is the required execution contract. It is an ordinary method and needs no decorator. @@ -1886,12 +1899,18 @@ operations: records a call, each action must be one the last recorded call made, by name and arguments (a `bash` call by its command alone, a whole float equal to its int), and all of them must run before the next `record` or the `reply`, else 409. - Calls to tools the turn doesn't offer aren't checked. `usage` is the call's, as the - model's response carried it, read by `Usage.from_model_response`, and its `chat` span takes - its token counts from that reading. The turn's `Usage`, on its reply or its error, sums each - count over the calls that gave it; its `provider_details` give `model_calls`, how many were - recorded, and `unreported`, how many left out each count only some gave, so such a sum reads - as a lower bound. + Calls to tools the turn doesn't offer aren't checked. `usage` is the call's: a `Usage` the + driver reports under `REPORTED_USAGE_KEY` (`{"agentenv_usage": usage.to_dict()}`), taken as + given, or else the model response's own, read by `Usage.from_model_response`. It rides in + `usage` rather than a field of its own, so a proxy on an earlier protocol still takes the + record. A reported `Usage` that isn't one gets 400 and leaves the turn as it was. The call's + input is made to count the cache, its `chat` span shows that reading, and its turn sums it. + The turn's `Usage`, on its reply or its error, sums each count over the calls that gave it, + `cost_usd` included. Its `provider_details` give `model_calls`, how many were recorded; + `unreported`, how many left out each count only some gave, so such a sum reads as a lower + bound; `cache_unsettled`, how many had cache counts with no word on whether their input held + them, which leaves the turn's flag unknown; `unrecognized`, how many usages were in no shape + the reader knows; and each provider's cache-write split, summed. - `reply` (`{"token", "task_id", "parts"}`, or `"error"` in place of `"parts"`) ends the turn: parts complete the task, and an error fails it as `proxy.driver_error`. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index af2817887..d679c4085 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -10,6 +10,7 @@ INSTALL_V1, MCP_CONFIG_V1, PEER_AGENTS_V1, + REPORTED_USAGE_KEY, SKILL_CONFIG_V1, SNAPSHOT_V1, STANDARD_EXTENSIONS, @@ -147,6 +148,7 @@ "INSTALL_V1", "MCP_CONFIG_V1", "PEER_AGENTS_V1", + "REPORTED_USAGE_KEY", "SKILL_CONFIG_V1", "SNAPSHOT_V1", "STAGING_ENDPOINT", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index eac35f346..06c91aba5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -450,6 +450,13 @@ class ProxyActionState(ExtensionResponse): result: Annotated[ProxyBashResult | ProxyToolResult, Field(discriminator="kind")] | None = None +# The key under which a record's ``usage`` carries a ``Usage`` its driver reported, as ``Usage.to_dict`` wrote it; any +# other ``usage`` is a model response's own, which the proxy reads with ``Usage.from_model_response``. It rides in +# ``usage`` rather than a field of its own so a proxy on an earlier protocol, whose request refuses unknown fields, +# still takes the record. +REPORTED_USAGE_KEY = "agentenv_usage" + + class ProxyRecordRequest(ExtensionRequest): token: str task_id: str diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py index e9e08a70d..9979b9081 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py @@ -19,7 +19,7 @@ import signal from collections import defaultdict from collections.abc import AsyncIterator -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field, fields, replace from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -31,6 +31,7 @@ from .extensions import ( MCP_CONFIG_V1, PROXY_V1, + REPORTED_USAGE_KEY, SKILL_CONFIG_V1, TRAJECTORY_V1, BundleSkillRequest, @@ -111,7 +112,7 @@ class _Turn: pending: list[tuple[str, str]] = field(default_factory=list) # (tool, its canonical arguments) not yet run last_event: str = field(default_factory=lambda: _now()) model: str | None = None - model_usage: list[Usage] = field(default_factory=list) # each recorded model call's + model_usage: list[Usage] = field(default_factory=list) # each recorded model call's, its input counting the cache @contextlib.asynccontextmanager @@ -239,6 +240,7 @@ async def result(self, request: ProxyResultRequest) -> ProxyActionState: @extension(PROXY_V1.record) async def record(self, request: ProxyRecordRequest) -> None: content, calls = _read_message(request.message) + usage = _counting_cache(_recorded_usage(request.usage)) offered = await self._tools(self._open_turn(request.token, request.task_id)) turn = self._open_turn(request.token, request.task_id) if turn.pending: @@ -248,7 +250,6 @@ async def record(self, request: ProxyRecordRequest) -> None: turn.recorded = True turn.model = request.model or turn.model span = _span("chat", "chat", turn.last_event, turn.model, completion={"content": content}) - usage = Usage.from_model_response(request.usage or {}) turn.model_usage.append(usage) for key in ("input_tokens", "output_tokens"): if (count := getattr(usage, key)) is not None: @@ -520,23 +521,64 @@ def _tool_error(message: str) -> ProxyToolResult: return ProxyToolResult(content=[{"type": "text", "text": f"error: {message}"}], is_error=True) -_USAGE_SUMS = tuple(f.name for f in fields(Usage) if f.name.endswith("_tokens")) +_USAGE_SUMS = (*(f.name for f in fields(Usage) if f.name.endswith("_tokens")), "cost_usd") +# Each provider's split of its cache writes by how long they're kept, which ``Usage.from_model_response`` keeps. +_CACHE_WRITE_SPLITS = ("cache_creation", "cache_write_tokens_details") + + +def _recorded_usage(usage: dict[str, Any] | None) -> Usage: + """A recorded call's usage: the ``Usage`` its driver reported under ``REPORTED_USAGE_KEY``, taken as given, or + else read from the model response's own by ``Usage.from_model_response``.""" + if usage is None: + return Usage() + if REPORTED_USAGE_KEY not in usage: + return Usage.from_model_response(usage) + try: + return Usage.from_dict(usage[REPORTED_USAGE_KEY]) + except (TypeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"usage[{REPORTED_USAGE_KEY!r}]: {e}") from e + + +def _counting_cache(usage: Usage) -> Usage: + """``usage`` with its ``input_tokens`` counting the cache, as its span shows it and its turn sums it: one that + leaves the cache out has it added, to its total too, and one with no cache to count counts it. One whose flag is + unknown while it has cache counts stays as given.""" + cache = (usage.cache_read_tokens or 0) + (usage.cache_write_tokens or 0) + if usage.input_tokens is None or usage.input_includes_cache is True: + return usage + if usage.input_includes_cache is False: + total = None if usage.total_tokens is None else usage.total_tokens + cache + return replace(usage, input_tokens=usage.input_tokens + cache, total_tokens=total, input_includes_cache=True) + return replace(usage, input_includes_cache=True) if cache == 0 else usage def _turn_usage(turn: _Turn) -> Usage: """The turn's usage: the calls to its tools, and each count summed over the recorded model calls that gave it, - None when none did. ``provider_details`` says how many calls were recorded and, for a count only some of them - gave, how many left it out, so a sum short of some calls reads as the lower bound it is.""" + None when none did, with ``input_tokens`` counting the cache. ``provider_details`` says how many calls were + recorded (``model_calls``); for a count only some gave, how many left it out (``unreported``), so the sum reads + as the lower bound it is; how many had cache counts with no word on whether their input held them + (``cache_unsettled``), which leaves the turn's flag unknown; how many usages were in a shape none of the readers + know (``unrecognized``); and each provider's split of the cache writes, summed.""" + calls = turn.model_usage sums, unreported = {}, {} for name in _USAGE_SUMS: - values = [value for call in turn.model_usage if (value := getattr(call, name)) is not None] + values = [value for call in calls if (value := getattr(call, name)) is not None] sums[name] = sum(values) if values else None - if values and len(values) < len(turn.model_usage): - unreported[name] = len(turn.model_usage) - len(values) - details = {"model_calls": len(turn.model_usage), **({"unreported": unreported} if unreported else {})} + if values and len(values) < len(calls): + unreported[name] = len(calls) - len(values) + unsettled = sum(call.input_tokens is not None and call.input_includes_cache is None for call in calls) + unrecognized = sum("unrecognized" in call.provider_details for call in calls) + splits: dict[str, dict[str, int]] = {} + for call in calls: + for name in _CACHE_WRITE_SPLITS: + for part, count in (call.provider_details.get(name) or {}).items(): + splits.setdefault(name, {})[part] = splits.get(name, {}).get(part, 0) + count + details = {"model_calls": len(calls), **({"unreported": unreported} if unreported else {}), + **({"cache_unsettled": unsettled} if unsettled else {}), + **({"unrecognized": unrecognized} if unrecognized else {}), **splits} return Usage(tool_call_count=len(turn.actions), **sums, - input_includes_cache=True if sums["input_tokens"] is not None else None, - provider_details=details if turn.model_usage else {}) + input_includes_cache=True if sums["input_tokens"] is not None and not unsettled else None, + provider_details=details if calls else {}) def _span(name: str, operation: str, started: str, model: str | None, *, prompt: Any = None, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 54f2a1a6d..1a0a2ab64 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -325,6 +325,13 @@ def __post_init__(self) -> None: ) +# The keys a model response's usage is read by, in any of the shapes ``Usage.from_model_response`` reads. +_RESPONSE_USAGE_KEYS = frozenset({ + "prompt_tokens", "completion_tokens", "input_tokens", "output_tokens", "total_tokens", + "prompt_tokens_details", "input_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens", +}) + + def _count(value: Any) -> int | None: return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None @@ -385,20 +392,25 @@ def from_dict(cls, data: Mapping[str, Any]) -> "Usage": @classmethod def from_model_response(cls, usage: Mapping[str, Any]) -> "Usage": - """A model call's usage as its response carried it, OpenAI's way or Anthropic's, with ``input_tokens`` - counting the cache. OpenAI's ``prompt_tokens`` counts what ``prompt_tokens_details.cached_tokens`` says was - cached, and so does a LiteLLM client's, which also counts ``cache_creation_input_tokens``; Anthropic's - ``input_tokens``, and the ``prompt_tokens`` a LiteLLM proxy sends with Anthropic's - ``cache_read_input_tokens`` and ``cache_creation_input_tokens`` beside it, count neither, so they're added. A - value that isn't a count is left out.""" - details = usage.get("prompt_tokens_details") - cached = _count(details.get("cached_tokens")) if isinstance(details, Mapping) else None + """A model call's usage as its response carried it, with ``input_tokens`` counting the cache: OpenAI's Chat + Completions (``prompt_tokens``) and Responses (``input_tokens``) shapes, and Anthropic's. An OpenAI count, + or a LiteLLM client's, says what it holds in its ``prompt_tokens_details`` or ``input_tokens_details``, whose + ``cached_tokens`` it counts, and ``cache_write_tokens`` with them. Anthropic's ``input_tokens``, and the + ``prompt_tokens`` a LiteLLM proxy sends with Anthropic's ``cache_read_input_tokens`` and + ``cache_creation_input_tokens`` beside it, count neither, so they're added, to ``total_tokens`` too. A + provider's split of its cache writes by how long they're kept (Anthropic's ``cache_creation``, OpenAI's + ``cache_write_tokens_details``), priced apart, is kept in ``provider_details``; so are the keys of a usage + in none of these shapes, as ``unrecognized``. A value that isn't a count is left out.""" + key = "prompt_tokens" if "prompt_tokens" in usage else "input_tokens" + details = usage.get(f"{key}_details") + details = details if isinstance(details, Mapping) else {} + cached = _count(details.get("cached_tokens")) read = cached if cached is not None else _count(usage.get("cache_read_input_tokens")) - write = _count(usage.get("cache_creation_input_tokens")) + written = _count(details.get("cache_write_tokens")) + write = written if written is not None else _count(usage.get("cache_creation_input_tokens")) cache = (read or 0) + (write or 0) - openai = "prompt_tokens" in usage - input_tokens = _count(usage["prompt_tokens"] if openai else usage.get("input_tokens")) - counts_cache = openai and cached is not None and input_tokens is not None and input_tokens >= cache + input_tokens = _count(usage.get(key)) + counts_cache = cached is not None and input_tokens is not None and input_tokens >= cache missing = cache if input_tokens is not None and not counts_cache else 0 if input_tokens is not None: input_tokens += missing @@ -407,8 +419,16 @@ def from_model_response(cls, usage: Mapping[str, Any]) -> "Usage": total += missing elif input_tokens is not None and output is not None: total = input_tokens + output + kept = {"cache_creation": usage.get("cache_creation"), + "cache_write_tokens_details": details.get("cache_write_tokens_details")} + provider_details: dict[str, Any] = { + name: counts for name, split in kept.items() if isinstance(split, Mapping) + if (counts := {k: v for k, v in split.items() if _count(v) is not None})} + if usage and _RESPONSE_USAGE_KEYS.isdisjoint(usage): + provider_details["unrecognized"] = sorted(map(str, usage)) return cls(input_tokens=input_tokens, output_tokens=output, total_tokens=total, cache_read_tokens=read, - cache_write_tokens=write, input_includes_cache=True if input_tokens is not None else None) + cache_write_tokens=write, input_includes_cache=True if input_tokens is not None else None, + provider_details=provider_details) @property def is_empty(self) -> bool: diff --git a/packages/agentenv-protocol/tests/test_a2a_proxy.py b/packages/agentenv-protocol/tests/test_a2a_proxy.py index e3bcb333f..65113897b 100644 --- a/packages/agentenv-protocol/tests/test_a2a_proxy.py +++ b/packages/agentenv-protocol/tests/test_a2a_proxy.py @@ -16,7 +16,7 @@ from starlette.testclient import TestClient from agentenv_protocol import EnvironmentCard, ToolContext, ToolSession, create_fastmcp_app, tool -from agentenv_protocol.a2a_agent import ProxyActionState, proxy +from agentenv_protocol.a2a_agent import REPORTED_USAGE_KEY, ProxyActionState, Usage, proxy from agentenv_protocol.a2a_agent.proxy import ProxyAgent TOKEN = "t" * 32 @@ -644,6 +644,72 @@ def test_a_turn_with_no_recorded_calls_reports_only_its_tool_calls(client): assert _usage(_settled(client, task_id)) == {"tool_call_count": 0} +def _reported(usage: Usage) -> dict[str, Any]: + return {REPORTED_USAGE_KEY: usage.to_dict()} + + +def test_a_usage_its_driver_reports_is_taken_as_given_its_input_made_to_count_the_cache(client): + task_id = _send(client, "go") + for usage in (_reported(Usage(input_tokens=9, output_tokens=5, total_tokens=14, cache_read_tokens=7809, + input_includes_cache=False, cost_usd=0.25)), + _reported(Usage(input_tokens=100, output_tokens=1, cache_read_tokens=60, input_includes_cache=True, + cost_usd=0.5)), + _reported(Usage(input_tokens=4, output_tokens=1))): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == { + "tool_call_count": 0, "input_tokens": 7818 + 100 + 4, "output_tokens": 7, "total_tokens": 7823, + "cache_read_tokens": 7869, "input_includes_cache": True, "cost_usd": 0.75, + "provider_details": {"model_calls": 3, "unreported": { + "total_tokens": 2, "cache_read_tokens": 1, "cost_usd": 1}}} + [chat, *_] = _trajectory(client, task_id) + assert chat["attributes"]["gen_ai.usage.input_tokens"] == 7818 + + +def test_a_reported_call_whose_input_may_or_may_not_hold_its_cache_leaves_the_turns_flag_unknown(client): + task_id = _send(client, "go") + _proxy(client, "record", task_id=task_id, message="Thinking.", + usage=_reported(Usage(input_tokens=9, output_tokens=5, cache_read_tokens=7809))) + _proxy(client, "record", task_id=task_id, message="Thinking.", usage={"prompt_tokens": 10, "completion_tokens": 1}) + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == { + "tool_call_count": 0, "input_tokens": 19, "output_tokens": 6, "total_tokens": 11, "cache_read_tokens": 7809, + "provider_details": {"model_calls": 2, "unreported": {"total_tokens": 1, "cache_read_tokens": 1}, + "cache_unsettled": 1}} + + +@pytest.mark.parametrize("reported", [{"input_tokens": -1}, {"input_includes_cache": "yes"}, ["input_tokens"], + {"provider_details": []}], ids=["negative", "flag", "not-a-mapping", "details"]) +def test_a_reported_usage_that_isnt_one_is_refused_and_the_turn_left_as_it_was(client, reported): + task_id = _send(client, "go") + + refused = _proxy(client, "record", task_id=task_id, message="Thinking.", usage={REPORTED_USAGE_KEY: reported}) + + assert refused.status_code == 400 and REPORTED_USAGE_KEY in refused.text + assert _proxy(client, "record", task_id=task_id, message="Thinking.", + usage={"input_tokens": 3, "output_tokens": 1}).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + assert _usage(_settled(client, task_id))["provider_details"] == {"model_calls": 1} + + +def test_a_turn_sums_each_providers_cache_write_split_and_counts_usage_it_couldnt_read(client): + task_id = _send(client, "go") + for usage in ({"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 7813, + "cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 0}}, + {"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 20, + "cache_creation": {"ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 20}}, + {"inputTokens": 5, "outputTokens": 2}): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id))["provider_details"] == { + "model_calls": 3, "unrecognized": 1, + "unreported": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 1, "cache_write_tokens": 1}, + "cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 20}} + + def test_an_env_tool_named_bash_is_held_to_all_its_recorded_arguments(terminal, monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) with TestClient(ProxyAgent(shell=False, long_poll_seconds=0.2).create_app()) as client: diff --git a/packages/agentenv-protocol/tests/test_usage.py b/packages/agentenv-protocol/tests/test_usage.py index eedd71378..f9f3553b4 100644 --- a/packages/agentenv-protocol/tests/test_usage.py +++ b/packages/agentenv-protocol/tests/test_usage.py @@ -73,8 +73,9 @@ def test_a_failed_result_carries_its_usage(): assert TaskResult.failure("agent.boom", "it broke").usage == Usage() -# Real responses to one Claude prompt of 7,818 tokens, 7,809 of them cacheable, and one GPT prompt, recorded through -# a LiteLLM proxy and, for the client's shape, as LiteLLM 1.83's client reads Anthropic's. +# Real responses recorded through a LiteLLM proxy: one Claude prompt of 7,818 tokens, 7,809 of them cacheable, and one +# GPT prompt, through Chat Completions, Anthropic's Messages and OpenAI's Responses; the LiteLLM client's shape is as +# LiteLLM 1.83's client reads Anthropic's, and the Responses read with a cache hit is OpenAI's documented shape. RESPONSES = { "litellm-proxy-claude-writes": ({"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14, "cache_creation_input_tokens": 7809, "cache_read_input_tokens": 0}, @@ -87,7 +88,31 @@ def test_a_failed_result_carries_its_usage(): "anthropic-messages-reads": ({"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 7809, "cache_creation": {"ephemeral_5m_input_tokens": 0}}, Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, cache_read_tokens=7809, - cache_write_tokens=0, input_includes_cache=True)), + cache_write_tokens=0, input_includes_cache=True, + provider_details={"cache_creation": {"ephemeral_5m_input_tokens": 0}})), + "anthropic-messages-writes-for-an-hour": ( + {"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 7813, "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 0}, "speed": "standard"}, + Usage(input_tokens=7822, output_tokens=5, total_tokens=7827, cache_read_tokens=0, cache_write_tokens=7813, + input_includes_cache=True, + provider_details={"cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 0}})), + "openai-responses-gpt": ( + {"input_tokens": 7823, "output_tokens": 3, "total_tokens": 7826, "cost": None, + "input_tokens_details": {"audio_tokens": None, "cached_tokens": 0, "cache_write_tokens": 0, "text_tokens": None, + "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 0}}, + "output_tokens_details": {"reasoning_tokens": 0, "text_tokens": None}}, + Usage(input_tokens=7823, output_tokens=3, total_tokens=7826, cache_read_tokens=0, cache_write_tokens=0, + input_includes_cache=True, + provider_details={"cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 0}})), + "openai-responses-claude": ( + {"input_tokens": 7819, "output_tokens": 5, "total_tokens": 7824, "cost": None, + "input_tokens_details": {"audio_tokens": None, "cached_tokens": 0, "text_tokens": 7819}, + "output_tokens_details": {"reasoning_tokens": 0, "text_tokens": 5}}, + Usage(input_tokens=7819, output_tokens=5, total_tokens=7824, cache_read_tokens=0, input_includes_cache=True)), + "openai-responses-reads": ({"input_tokens": 7823, "output_tokens": 3, "total_tokens": 7826, + "input_tokens_details": {"cached_tokens": 7680}}, + Usage(input_tokens=7823, output_tokens=3, total_tokens=7826, cache_read_tokens=7680, + input_includes_cache=True)), "litellm-client-claude-reads": ({"prompt_tokens": 7818, "completion_tokens": 5, "total_tokens": 7823, "prompt_tokens_details": {"cached_tokens": 7809}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 7809}, @@ -104,6 +129,9 @@ def test_a_failed_result_carries_its_usage(): input_includes_cache=True)), "nothing": ({}, Usage()), "unreadable": ({"prompt_tokens": "9", "completion_tokens": True, "prompt_tokens_details": "x"}, Usage()), + "bedrock-camel-case": ({"inputTokens": 5, "outputTokens": 2, "cacheReadInputTokens": 1}, + Usage(provider_details={"unrecognized": ["cacheReadInputTokens", "inputTokens", + "outputTokens"]})), } diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index fe02fd97e..a25b5086d 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -54,8 +54,9 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn. text part, so add one. - **Usage:** when the runtime counts prompt-cache tokens, put them in `Usage`'s `cache_read_tokens` and `cache_write_tokens`, not `provider_details`, and set `input_includes_cache` to whether `input_tokens` counts them - (`None` when it can't tell). Code that sets them needs a `requirements.txt` protocol floor at the first release that - has them. + (`None` when it can't tell). Set it per call: through a gateway it depends on the model behind it. Where the runtime + hands back a model response's own usage, `Usage.from_model_response` reads it. Code that sets these fields needs a + `requirements.txt` protocol floor at the first release that has them. - **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text never sent. diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index 0de563ffb..b9b446789 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -8,11 +8,13 @@ import httpx from agentenv_protocol import EnvironmentTool, ToolResult from agentenv_protocol.a2a_agent import ( + REPORTED_USAGE_KEY, ProxyBashAction, ProxyBashResult, ProxySkill, ProxyTextPart, ProxyToolAction, + Usage, ) from agent_env.a2a_agent import protocol @@ -66,11 +68,14 @@ def tools(self, *, timeout: float = 600.0) -> TurnTools: return TurnTools(self, timeout) async def record_model_call(self, message: str | dict[str, Any], *, model: str | None = None, - usage: dict[str, Any] | None = None) -> None: + usage: Usage | dict[str, Any] | None = None) -> None: """Record one of the model's calls, once it has returned: an OpenAI chat message, an Anthropic message or a string. From then on each call to the turn's tools must be one the last recorded call made, and all must run before the next is recorded or the turn ends; else ``EpisodeError``. Calls to tools the turn doesn't offer - aren't checked.""" + aren't checked. ``usage`` is the call's: a ``Usage``, taken as given, from a caller that knows what its counts + hold, or the response's own usage, which the proxy reads with ``Usage.from_model_response``.""" + if isinstance(usage, Usage): + usage = {REPORTED_USAGE_KEY: usage.to_dict()} await self._checked(self._proxy.record(self._task_id, message, model, usage)) async def end(self, response: str | None = None) -> None: diff --git a/tst/integration/agent/proxy_usage_local_test.py b/tst/integration/agent/proxy_usage_local_test.py index 0f52422b4..3a46ef5a0 100644 --- a/tst/integration/agent/proxy_usage_local_test.py +++ b/tst/integration/agent/proxy_usage_local_test.py @@ -1,6 +1,7 @@ """A played agent's turn reaches the host with the usage of the model calls its trainer recorded: an episode's proxy, -served in this process, sums an OpenAI-shaped and an Anthropic-shaped call into one ``Usage`` whose ``input_tokens`` -counts the cache, and the host stores it with the agent's reply in the run's conversation. Local stores, no Docker.""" +served in this process, sums an OpenAI-shaped and an Anthropic-shaped response's usage and a ``Usage`` the trainer +reports itself into one ``Usage`` whose ``input_tokens`` counts the cache, and the host stores it with the agent's reply +in the run's conversation. Local stores, no Docker.""" import pytest from agentenv_protocol.a2a_agent.tasks.v1 import Usage @@ -14,6 +15,8 @@ OPENAI = {"prompt_tokens": 120, "completion_tokens": 7, "total_tokens": 127, "prompt_tokens_details": {"cached_tokens": 100}} ANTHROPIC = {"input_tokens": 3, "output_tokens": 9, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20} +REPORTED = Usage(input_tokens=4, output_tokens=2, total_tokens=6, cache_read_tokens=10, input_includes_cache=False, + cost_usd=0.02) @pytest.fixture @@ -51,7 +54,9 @@ async def test_a_turns_recorded_model_calls_reach_the_host_as_the_turns_usage(lo turn = await episode.next_turn() await turn.record_model_call({"role": "assistant", "content": "Thinking."}, model="policy-model", usage=OPENAI) - await turn.record_model_call("Hi.", model="policy-model", usage=ANTHROPIC) + await turn.record_model_call({"role": "assistant", "content": "Still thinking."}, model="policy-model", + usage=ANTHROPIC) + await turn.record_model_call("Hi.", model="policy-model", usage=REPORTED) await turn.end("Hi.") await episode.result() finally: @@ -62,7 +67,7 @@ async def test_a_turns_recorded_model_calls_reach_the_host_as_the_turns_usage(lo [play] = record.context["prompt_responses"] assert play["response"] == "Hi." [usage] = _stored_usage(play["a2a_context_id"]) - assert Usage.from_dict(usage) == Usage(tool_call_count=0, input_tokens=120 + 3 + 100 + 20, output_tokens=16, - total_tokens=127 + 123 + 9, cache_read_tokens=200, cache_write_tokens=20, - input_includes_cache=True, - provider_details={"model_calls": 2, "unreported": {"cache_write_tokens": 1}}) + assert Usage.from_dict(usage) == Usage( + tool_call_count=0, input_tokens=120 + 123 + 14, output_tokens=18, total_tokens=127 + 132 + 16, + cache_read_tokens=210, cache_write_tokens=20, input_includes_cache=True, cost_usd=0.02, + provider_details={"model_calls": 3, "unreported": {"cache_write_tokens": 2, "cost_usd": 2}}) From 4e9407f8d7941b49703ab542a1f817d59ea38bc7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 18 Nov 2026 20:44:44 -0800 Subject: [PATCH 7/7] fix(protocol): a reported Usage's cache-write split is summed only where it holds counts A driver's Usage may carry any provider_details, so a split with a None count no longer fails the turn when it ends. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/a2a_agent/proxy.py | 11 +++++++---- .../agentenv-protocol/tests/test_a2a_proxy.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py index 9979b9081..1b0a42ba9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py @@ -18,7 +18,7 @@ import shutil import signal from collections import defaultdict -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass, field, fields, replace from datetime import datetime, timezone from pathlib import Path @@ -558,7 +558,8 @@ def _turn_usage(turn: _Turn) -> Usage: recorded (``model_calls``); for a count only some gave, how many left it out (``unreported``), so the sum reads as the lower bound it is; how many had cache counts with no word on whether their input held them (``cache_unsettled``), which leaves the turn's flag unknown; how many usages were in a shape none of the readers - know (``unrecognized``); and each provider's split of the cache writes, summed.""" + know (``unrecognized``); and each provider's split of the cache writes, summed, a reported ``Usage``'s split + counted only where it holds counts, as its ``provider_details`` may hold anything.""" calls = turn.model_usage sums, unreported = {}, {} for name in _USAGE_SUMS: @@ -571,8 +572,10 @@ def _turn_usage(turn: _Turn) -> Usage: splits: dict[str, dict[str, int]] = {} for call in calls: for name in _CACHE_WRITE_SPLITS: - for part, count in (call.provider_details.get(name) or {}).items(): - splits.setdefault(name, {})[part] = splits.get(name, {}).get(part, 0) + count + if isinstance(split := call.provider_details.get(name), Mapping): + for part, count in split.items(): + if isinstance(count, int) and not isinstance(count, bool) and count >= 0: + splits.setdefault(name, {})[part] = splits.get(name, {}).get(part, 0) + count details = {"model_calls": len(calls), **({"unreported": unreported} if unreported else {}), **({"cache_unsettled": unsettled} if unsettled else {}), **({"unrecognized": unrecognized} if unrecognized else {}), **splits} diff --git a/packages/agentenv-protocol/tests/test_a2a_proxy.py b/packages/agentenv-protocol/tests/test_a2a_proxy.py index 65113897b..b37bc69fe 100644 --- a/packages/agentenv-protocol/tests/test_a2a_proxy.py +++ b/packages/agentenv-protocol/tests/test_a2a_proxy.py @@ -694,6 +694,21 @@ def test_a_reported_usage_that_isnt_one_is_refused_and_the_turn_left_as_it_was(c assert _usage(_settled(client, task_id))["provider_details"] == {"model_calls": 1} +def test_a_reported_usages_cache_write_split_counts_only_where_it_holds_counts(client): + task_id = _send(client, "go") + for details in ({"cache_creation": {"ephemeral_1h_input_tokens": None, "ephemeral_5m_input_tokens": 7}}, + {"cache_creation": "x", "cache_write_tokens_details": [1, 2]}, + {"cache_creation": {"ephemeral_5m_input_tokens": "3", "ephemeral_1h_input_tokens": True, + "other": -1}}): + usage = _reported(Usage(input_tokens=1, output_tokens=1, provider_details=details)) + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + task = _settled(client, task_id) + assert task["status"]["state"] == "completed" + assert _usage(task)["provider_details"] == {"model_calls": 3, "cache_creation": {"ephemeral_5m_input_tokens": 7}} + + def test_a_turn_sums_each_providers_cache_write_split_and_counts_usage_it_couldnt_read(client): task_id = _send(client, "go") for usage in ({"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 7813,