diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index c821ea5e3..09e21129b 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1388,6 +1388,39 @@ Extensions come from explicit definitions, configured activations, and decorated handlers in one validated registry, so routes and card advertisement cannot drift. +`Usage` reports what a task used: `tool_call_count`, `input_tokens`, +`output_tokens`, `total_tokens`, `cost_usd`, the prompt-cache counts +`cache_read_tokens` and `cache_write_tokens`, and `input_includes_cache`, which +says whether `input_tokens` counts both cache counts (`True`), neither +(`False`), or is unknown (`None`, the default). Set the flag whenever you set +the cache counts: a price can't read them without it. Runtime-specific counts go +in `provider_details`, but cache counts go in their own fields. `Usage.from_dict` +reads what `to_dict` wrote under any protocol version, a field it leaves out +being `None` and a key it doesn't know ignored, and `TaskResult.failure(..., +usage=...)` carries usage on a failure as `success` does. + +Whoever makes a model call knows what its counts hold, so an agent builds its own +`Usage`. `Usage.from_model_response(usage)` is the default reader for a model +response's own usage, with `input_tokens` counting the cache +(`input_includes_cache` true), for OpenAI's Chat Completions (`prompt_tokens`) +and Responses (`input_tokens`) shapes and Anthropic's: + +- An OpenAI count, or a LiteLLM client's, counts the cache its + `prompt_tokens_details` or `input_tokens_details` names: `cached_tokens`, and + `cache_write_tokens` beside it. +- Anthropic's `input_tokens` counts neither `cache_read_input_tokens` nor + `cache_creation_input_tokens`, and neither does the `prompt_tokens` a LiteLLM + proxy sends with those two beside it, so they're added, to `total_tokens` too. +- A provider's split of its cache writes by how long they're kept, priced apart + (Anthropic's `cache_creation`, OpenAI's `cache_write_tokens_details`), is kept + in `provider_details`, and so are the keys of a usage in none of these shapes, + as `unrecognized`. A value that isn't a count is left out. + +Through a gateway, whether a response's input counts the cache depends on the +model behind it: a LiteLLM proxy's Claude answers through Chat Completions leave +it out, its GPT answers don't. Set `input_includes_cache` per call, never once +per agent. + `run(request)` is the required execution contract. It is an ordinary method and needs no decorator. @@ -1866,7 +1899,18 @@ operations: records a call, each action must be one the last recorded call made, by name and arguments (a `bash` call by its command alone, a whole float equal to its int), and all of them must run before the next `record` or the `reply`, else 409. - Calls to tools the turn doesn't offer aren't checked. + Calls to tools the turn doesn't offer aren't checked. `usage` is the call's: a `Usage` the + driver reports under `REPORTED_USAGE_KEY` (`{"agentenv_usage": usage.to_dict()}`), taken as + given, or else the model response's own, read by `Usage.from_model_response`. It rides in + `usage` rather than a field of its own, so a proxy on an earlier protocol still takes the + record. A reported `Usage` that isn't one gets 400 and leaves the turn as it was. The call's + input is made to count the cache, its `chat` span shows that reading, and its turn sums it. + The turn's `Usage`, on its reply or its error, sums each count over the calls that gave it, + `cost_usd` included. Its `provider_details` give `model_calls`, how many were recorded; + `unreported`, how many left out each count only some gave, so such a sum reads as a lower + bound; `cache_unsettled`, how many had cache counts with no word on whether their input held + them, which leaves the turn's flag unknown; `unrecognized`, how many usages were in no shape + the reader knows; and each provider's cache-write split, summed. - `reply` (`{"token", "task_id", "parts"}`, or `"error"` in place of `"parts"`) ends the turn: parts complete the task, and an error fails it as `proxy.driver_error`. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index af2817887..d679c4085 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -10,6 +10,7 @@ INSTALL_V1, MCP_CONFIG_V1, PEER_AGENTS_V1, + REPORTED_USAGE_KEY, SKILL_CONFIG_V1, SNAPSHOT_V1, STANDARD_EXTENSIONS, @@ -147,6 +148,7 @@ "INSTALL_V1", "MCP_CONFIG_V1", "PEER_AGENTS_V1", + "REPORTED_USAGE_KEY", "SKILL_CONFIG_V1", "SNAPSHOT_V1", "STAGING_ENDPOINT", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py index eac35f346..06c91aba5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/extensions.py @@ -450,6 +450,13 @@ class ProxyActionState(ExtensionResponse): result: Annotated[ProxyBashResult | ProxyToolResult, Field(discriminator="kind")] | None = None +# The key under which a record's ``usage`` carries a ``Usage`` its driver reported, as ``Usage.to_dict`` wrote it; any +# other ``usage`` is a model response's own, which the proxy reads with ``Usage.from_model_response``. It rides in +# ``usage`` rather than a field of its own so a proxy on an earlier protocol, whose request refuses unknown fields, +# still takes the record. +REPORTED_USAGE_KEY = "agentenv_usage" + + class ProxyRecordRequest(ExtensionRequest): token: str task_id: str diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py index 15f5189f9..1b0a42ba9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/proxy.py @@ -18,8 +18,8 @@ import shutil import signal from collections import defaultdict -from collections.abc import AsyncIterator -from dataclasses import dataclass, field +from collections.abc import AsyncIterator, Mapping +from dataclasses import dataclass, field, fields, replace from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -31,6 +31,7 @@ from .extensions import ( MCP_CONFIG_V1, PROXY_V1, + REPORTED_USAGE_KEY, SKILL_CONFIG_V1, TRAJECTORY_V1, BundleSkillRequest, @@ -111,6 +112,7 @@ class _Turn: pending: list[tuple[str, str]] = field(default_factory=list) # (tool, its canonical arguments) not yet run last_event: str = field(default_factory=lambda: _now()) model: str | None = None + model_usage: list[Usage] = field(default_factory=list) # each recorded model call's, its input counting the cache @contextlib.asynccontextmanager @@ -174,11 +176,11 @@ async def run(self, request: TaskRequest[ProxyConfig]) -> TaskResult: if reply.error is not None: turn.trajectory.append(_span("turn", "chain", turn.started, turn.model, prompt=prompt, completion={"error": reply.error}, failed=True)) - return TaskResult.failure("proxy.driver_error", reply.error) + return TaskResult.failure("proxy.driver_error", reply.error, usage=_turn_usage(turn)) turn.trajectory.append(_span("turn", "chain", turn.started, turn.model, prompt=prompt, completion={ "content": [{"type": "text", "text": part.text} for part in reply.parts]})) return TaskResult.success(parts=tuple(TextPart(text=part.text) for part in reply.parts), - usage=Usage(tool_call_count=len(turn.actions))) + usage=_turn_usage(turn)) @extension(PROXY_V1.claim) async def claim(self, request: ProxyClaimRequest) -> None: @@ -238,6 +240,7 @@ async def result(self, request: ProxyResultRequest) -> ProxyActionState: @extension(PROXY_V1.record) async def record(self, request: ProxyRecordRequest) -> None: content, calls = _read_message(request.message) + usage = _counting_cache(_recorded_usage(request.usage)) offered = await self._tools(self._open_turn(request.token, request.task_id)) turn = self._open_turn(request.token, request.task_id) if turn.pending: @@ -247,10 +250,9 @@ async def record(self, request: ProxyRecordRequest) -> None: turn.recorded = True turn.model = request.model or turn.model span = _span("chat", "chat", turn.last_event, turn.model, completion={"content": content}) - usage = request.usage or {} - for key, names in (("input_tokens", ("input_tokens", "prompt_tokens")), - ("output_tokens", ("output_tokens", "completion_tokens"))): - if (count := next((usage[name] for name in names if name in usage), None)) is not None: + turn.model_usage.append(usage) + for key in ("input_tokens", "output_tokens"): + if (count := getattr(usage, key)) is not None: span["attributes"][f"gen_ai.usage.{key}"] = count turn.trajectory.append(span) turn.last_event = span["end_time"] @@ -519,6 +521,69 @@ def _tool_error(message: str) -> ProxyToolResult: return ProxyToolResult(content=[{"type": "text", "text": f"error: {message}"}], is_error=True) +_USAGE_SUMS = (*(f.name for f in fields(Usage) if f.name.endswith("_tokens")), "cost_usd") +# Each provider's split of its cache writes by how long they're kept, which ``Usage.from_model_response`` keeps. +_CACHE_WRITE_SPLITS = ("cache_creation", "cache_write_tokens_details") + + +def _recorded_usage(usage: dict[str, Any] | None) -> Usage: + """A recorded call's usage: the ``Usage`` its driver reported under ``REPORTED_USAGE_KEY``, taken as given, or + else read from the model response's own by ``Usage.from_model_response``.""" + if usage is None: + return Usage() + if REPORTED_USAGE_KEY not in usage: + return Usage.from_model_response(usage) + try: + return Usage.from_dict(usage[REPORTED_USAGE_KEY]) + except (TypeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"usage[{REPORTED_USAGE_KEY!r}]: {e}") from e + + +def _counting_cache(usage: Usage) -> Usage: + """``usage`` with its ``input_tokens`` counting the cache, as its span shows it and its turn sums it: one that + leaves the cache out has it added, to its total too, and one with no cache to count counts it. One whose flag is + unknown while it has cache counts stays as given.""" + cache = (usage.cache_read_tokens or 0) + (usage.cache_write_tokens or 0) + if usage.input_tokens is None or usage.input_includes_cache is True: + return usage + if usage.input_includes_cache is False: + total = None if usage.total_tokens is None else usage.total_tokens + cache + return replace(usage, input_tokens=usage.input_tokens + cache, total_tokens=total, input_includes_cache=True) + return replace(usage, input_includes_cache=True) if cache == 0 else usage + + +def _turn_usage(turn: _Turn) -> Usage: + """The turn's usage: the calls to its tools, and each count summed over the recorded model calls that gave it, + None when none did, with ``input_tokens`` counting the cache. ``provider_details`` says how many calls were + recorded (``model_calls``); for a count only some gave, how many left it out (``unreported``), so the sum reads + as the lower bound it is; how many had cache counts with no word on whether their input held them + (``cache_unsettled``), which leaves the turn's flag unknown; how many usages were in a shape none of the readers + know (``unrecognized``); and each provider's split of the cache writes, summed, a reported ``Usage``'s split + counted only where it holds counts, as its ``provider_details`` may hold anything.""" + calls = turn.model_usage + sums, unreported = {}, {} + for name in _USAGE_SUMS: + values = [value for call in calls if (value := getattr(call, name)) is not None] + sums[name] = sum(values) if values else None + if values and len(values) < len(calls): + unreported[name] = len(calls) - len(values) + unsettled = sum(call.input_tokens is not None and call.input_includes_cache is None for call in calls) + unrecognized = sum("unrecognized" in call.provider_details for call in calls) + splits: dict[str, dict[str, int]] = {} + for call in calls: + for name in _CACHE_WRITE_SPLITS: + if isinstance(split := call.provider_details.get(name), Mapping): + for part, count in split.items(): + if isinstance(count, int) and not isinstance(count, bool) and count >= 0: + splits.setdefault(name, {})[part] = splits.get(name, {}).get(part, 0) + count + details = {"model_calls": len(calls), **({"unreported": unreported} if unreported else {}), + **({"cache_unsettled": unsettled} if unsettled else {}), + **({"unrecognized": unrecognized} if unrecognized else {}), **splits} + return Usage(tool_call_count=len(turn.actions), **sums, + input_includes_cache=True if sums["input_tokens"] is not None and not unsettled else None, + provider_details=details if calls else {}) + + def _span(name: str, operation: str, started: str, model: str | None, *, prompt: Any = None, completion: Any = None, failed: bool = False) -> dict[str, Any]: """An OTel GenAI span, the trajectory format rubrics and the explorer read.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 0634de182..1a0a2ab64 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -11,7 +11,7 @@ import json import logging from collections.abc import AsyncIterator, Awaitable, Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from enum import Enum from pathlib import Path from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union @@ -315,29 +315,55 @@ def __post_init__(self) -> None: object.__setattr__(self, "payload", _copy_json_value(self.payload)) +_USAGE_COUNTERS = ( + "tool_call_count", + "input_tokens", + "output_tokens", + "total_tokens", + "cache_read_tokens", + "cache_write_tokens", +) + + +# The keys a model response's usage is read by, in any of the shapes ``Usage.from_model_response`` reads. +_RESPONSE_USAGE_KEYS = frozenset({ + "prompt_tokens", "completion_tokens", "input_tokens", "output_tokens", "total_tokens", + "prompt_tokens_details", "input_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens", +}) + + +def _count(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + @dataclass(frozen=True, slots=True, kw_only=True) class Usage: - """Common execution metrics plus runtime-specific, JSON-shaped details.""" + """Common execution metrics plus runtime-specific, JSON-shaped details. ``cache_read_tokens`` and + ``cache_write_tokens`` count the prompt tokens read from and written to the provider's prompt cache; + ``input_includes_cache`` says whether ``input_tokens`` counts both of them (True), neither (False), or is unknown + (None), which a price needs to read the counts.""" tool_call_count: int | None = None input_tokens: int | None = None output_tokens: int | None = None total_tokens: int | None = None + cache_read_tokens: int | None = None + cache_write_tokens: int | None = None + input_includes_cache: bool | None = None cost_usd: float | None = None provider_details: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: - for name in ( - "tool_call_count", - "input_tokens", - "output_tokens", - "total_tokens", - ): + for name in _USAGE_COUNTERS: value = getattr(self, name) if value is not None and ( isinstance(value, bool) or not isinstance(value, int) or value < 0 ): - raise ValueError(f"{name} cannot be negative") + raise ValueError(f"{name} must be a non-negative integer") + if self.input_includes_cache is not None and not isinstance( + self.input_includes_cache, bool + ): + raise ValueError("input_includes_cache must be true, false or null") if self.cost_usd is not None and ( isinstance(self.cost_usd, bool) or not isinstance(self.cost_usd, (int, float)) @@ -350,30 +376,74 @@ def __post_init__(self) -> None: self, "provider_details", _copy_json_value(self.provider_details) ) + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "Usage": + """The usage ``to_dict`` wrote, from this protocol version or another: a field it leaves out is None, and a + key this version doesn't know is ignored. Raises ValueError or TypeError for a field it can't hold.""" + if not isinstance(data, Mapping): + raise TypeError("usage must be a mapping") + known = { + f.name: data[f.name] + for f in fields(cls) + if f.name != "provider_details" and data.get(f.name) is not None + } + details = data.get("provider_details") + return cls(**known, provider_details={} if details is None else details) + + @classmethod + def from_model_response(cls, usage: Mapping[str, Any]) -> "Usage": + """A model call's usage as its response carried it, with ``input_tokens`` counting the cache: OpenAI's Chat + Completions (``prompt_tokens``) and Responses (``input_tokens``) shapes, and Anthropic's. An OpenAI count, + or a LiteLLM client's, says what it holds in its ``prompt_tokens_details`` or ``input_tokens_details``, whose + ``cached_tokens`` it counts, and ``cache_write_tokens`` with them. Anthropic's ``input_tokens``, and the + ``prompt_tokens`` a LiteLLM proxy sends with Anthropic's ``cache_read_input_tokens`` and + ``cache_creation_input_tokens`` beside it, count neither, so they're added, to ``total_tokens`` too. A + provider's split of its cache writes by how long they're kept (Anthropic's ``cache_creation``, OpenAI's + ``cache_write_tokens_details``), priced apart, is kept in ``provider_details``; so are the keys of a usage + in none of these shapes, as ``unrecognized``. A value that isn't a count is left out.""" + key = "prompt_tokens" if "prompt_tokens" in usage else "input_tokens" + details = usage.get(f"{key}_details") + details = details if isinstance(details, Mapping) else {} + cached = _count(details.get("cached_tokens")) + read = cached if cached is not None else _count(usage.get("cache_read_input_tokens")) + written = _count(details.get("cache_write_tokens")) + write = written if written is not None else _count(usage.get("cache_creation_input_tokens")) + cache = (read or 0) + (write or 0) + input_tokens = _count(usage.get(key)) + counts_cache = cached is not None and input_tokens is not None and input_tokens >= cache + missing = cache if input_tokens is not None and not counts_cache else 0 + if input_tokens is not None: + input_tokens += missing + output = _count(usage.get("completion_tokens", usage.get("output_tokens"))) + if (total := _count(usage.get("total_tokens"))) is not None: + total += missing + elif input_tokens is not None and output is not None: + total = input_tokens + output + kept = {"cache_creation": usage.get("cache_creation"), + "cache_write_tokens_details": details.get("cache_write_tokens_details")} + provider_details: dict[str, Any] = { + name: counts for name, split in kept.items() if isinstance(split, Mapping) + if (counts := {k: v for k, v in split.items() if _count(v) is not None})} + if usage and _RESPONSE_USAGE_KEYS.isdisjoint(usage): + provider_details["unrecognized"] = sorted(map(str, usage)) + return cls(input_tokens=input_tokens, output_tokens=output, total_tokens=total, cache_read_tokens=read, + cache_write_tokens=write, input_includes_cache=True if input_tokens is not None else None, + provider_details=provider_details) + @property def is_empty(self) -> bool: - return ( - self.tool_call_count is None - and self.input_tokens is None - and self.output_tokens is None - and self.total_tokens is None - and self.cost_usd is None - and not self.provider_details - ) + return all(value is None for value in self._values().values()) and not self.provider_details def to_dict(self) -> dict[str, Any]: - values = { - "tool_call_count": self.tool_call_count, - "input_tokens": self.input_tokens, - "output_tokens": self.output_tokens, - "total_tokens": self.total_tokens, - "cost_usd": self.cost_usd, - } - result = {name: value for name, value in values.items() if value is not None} + result = {name: value for name, value in self._values().items() if value is not None} if self.provider_details: result["provider_details"] = _thaw(self.provider_details) return result + def _values(self) -> dict[str, Any]: + """Every field but ``provider_details``, by name, in declaration order.""" + return {f.name: getattr(self, f.name) for f in fields(self) if f.name != "provider_details"} + @dataclass(frozen=True, slots=True, kw_only=True) class TaskResult: @@ -438,11 +508,13 @@ def failure( *, error_type: Literal["agent_error", "infra_error"] = "agent_error", parts: Iterable[TaskPart] | None = None, + usage: Usage | None = None, ) -> "TaskResult": return ( cls.builder() .failed(code, message, error_type=error_type) .parts(parts if parts is not None else (TextPart(text=message),)) + .usage(usage or Usage()) .build() ) diff --git a/packages/agentenv-protocol/tests/test_a2a_proxy.py b/packages/agentenv-protocol/tests/test_a2a_proxy.py index 43450baba..b37bc69fe 100644 --- a/packages/agentenv-protocol/tests/test_a2a_proxy.py +++ b/packages/agentenv-protocol/tests/test_a2a_proxy.py @@ -16,7 +16,7 @@ from starlette.testclient import TestClient from agentenv_protocol import EnvironmentCard, ToolContext, ToolSession, create_fastmcp_app, tool -from agentenv_protocol.a2a_agent import ProxyActionState, proxy +from agentenv_protocol.a2a_agent import REPORTED_USAGE_KEY, ProxyActionState, Usage, proxy from agentenv_protocol.a2a_agent.proxy import ProxyAgent TOKEN = "t" * 32 @@ -578,6 +578,153 @@ def test_recorded_model_calls_are_checked_against_the_actions_that_run(client, e assert task["status"]["message"]["parts"][1]["data"]["usage"]["tool_call_count"] == 3 +def _usage(task: dict[str, Any]) -> dict[str, Any]: + [usage] = [part["data"]["usage"] for part in task["status"]["message"]["parts"] + if part.get("kind") == "data" and "usage" in part["data"]] + return usage + + +def test_a_turns_usage_sums_its_recorded_calls_with_input_counting_the_cache(client): + task_id = _send(client, "go") + for usage in ({"prompt_tokens": 120, "completion_tokens": 7, "total_tokens": 127, + "prompt_tokens_details": {"cached_tokens": 100}}, + {"input_tokens": 3, "output_tokens": 9, "cache_read_input_tokens": 100, + "cache_creation_input_tokens": 20}, + {"prompt_tokens": 50, "completion_tokens": 1, "cache_read_input_tokens": 40, + "cache_creation_input_tokens": 10}, + None): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == { + "tool_call_count": 0, "input_tokens": 120 + 123 + 100, "output_tokens": 17, "total_tokens": 127 + 132 + 101, + "cache_read_tokens": 240, "cache_write_tokens": 30, "input_includes_cache": True, + "provider_details": {"model_calls": 4, "unreported": { + "input_tokens": 1, "output_tokens": 1, "total_tokens": 1, "cache_read_tokens": 1, "cache_write_tokens": 2}}} + + +def test_counts_a_recorded_call_doesnt_give_or_cant_be_read_are_left_out(client): + task_id = _send(client, "go") + for usage in ({"prompt_tokens": "9", "completion_tokens": True, "prompt_tokens_details": "none"}, + {"completion_tokens": 4}): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == {"tool_call_count": 0, "output_tokens": 4, "provider_details": { + "model_calls": 2, "unreported": {"output_tokens": 1}}} + + +def test_an_error_reply_keeps_the_usage_recorded_before_it(client): + task_id = _send(client, "go") + _proxy(client, "record", task_id=task_id, message="Thinking.", usage={"input_tokens": 5, "output_tokens": 2}) + _proxy(client, "reply", task_id=task_id, error="the driver gave up") + + task = _settled(client, task_id) + assert task["status"]["state"] == "failed" + assert _usage(task) == {"tool_call_count": 0, "input_tokens": 5, "output_tokens": 2, "total_tokens": 7, + "input_includes_cache": True, "provider_details": {"model_calls": 1}} + + +def test_a_chat_spans_counts_are_the_ones_the_turns_usage_sums(client): + task_id = _send(client, "go") + _proxy(client, "record", task_id=task_id, message="Thinking.", usage={ + "prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14, "cache_read_input_tokens": 7809}) + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + usage = _usage(_settled(client, task_id)) + [chat, _] = _trajectory(client, task_id) + assert (chat["attributes"]["gen_ai.usage.input_tokens"], chat["attributes"]["gen_ai.usage.output_tokens"]) == ( + usage["input_tokens"], usage["output_tokens"]) == (7818, 5) + + +def test_a_turn_with_no_recorded_calls_reports_only_its_tool_calls(client): + task_id = _send(client, "go") + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == {"tool_call_count": 0} + + +def _reported(usage: Usage) -> dict[str, Any]: + return {REPORTED_USAGE_KEY: usage.to_dict()} + + +def test_a_usage_its_driver_reports_is_taken_as_given_its_input_made_to_count_the_cache(client): + task_id = _send(client, "go") + for usage in (_reported(Usage(input_tokens=9, output_tokens=5, total_tokens=14, cache_read_tokens=7809, + input_includes_cache=False, cost_usd=0.25)), + _reported(Usage(input_tokens=100, output_tokens=1, cache_read_tokens=60, input_includes_cache=True, + cost_usd=0.5)), + _reported(Usage(input_tokens=4, output_tokens=1))): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == { + "tool_call_count": 0, "input_tokens": 7818 + 100 + 4, "output_tokens": 7, "total_tokens": 7823, + "cache_read_tokens": 7869, "input_includes_cache": True, "cost_usd": 0.75, + "provider_details": {"model_calls": 3, "unreported": { + "total_tokens": 2, "cache_read_tokens": 1, "cost_usd": 1}}} + [chat, *_] = _trajectory(client, task_id) + assert chat["attributes"]["gen_ai.usage.input_tokens"] == 7818 + + +def test_a_reported_call_whose_input_may_or_may_not_hold_its_cache_leaves_the_turns_flag_unknown(client): + task_id = _send(client, "go") + _proxy(client, "record", task_id=task_id, message="Thinking.", + usage=_reported(Usage(input_tokens=9, output_tokens=5, cache_read_tokens=7809))) + _proxy(client, "record", task_id=task_id, message="Thinking.", usage={"prompt_tokens": 10, "completion_tokens": 1}) + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id)) == { + "tool_call_count": 0, "input_tokens": 19, "output_tokens": 6, "total_tokens": 11, "cache_read_tokens": 7809, + "provider_details": {"model_calls": 2, "unreported": {"total_tokens": 1, "cache_read_tokens": 1}, + "cache_unsettled": 1}} + + +@pytest.mark.parametrize("reported", [{"input_tokens": -1}, {"input_includes_cache": "yes"}, ["input_tokens"], + {"provider_details": []}], ids=["negative", "flag", "not-a-mapping", "details"]) +def test_a_reported_usage_that_isnt_one_is_refused_and_the_turn_left_as_it_was(client, reported): + task_id = _send(client, "go") + + refused = _proxy(client, "record", task_id=task_id, message="Thinking.", usage={REPORTED_USAGE_KEY: reported}) + + assert refused.status_code == 400 and REPORTED_USAGE_KEY in refused.text + assert _proxy(client, "record", task_id=task_id, message="Thinking.", + usage={"input_tokens": 3, "output_tokens": 1}).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + assert _usage(_settled(client, task_id))["provider_details"] == {"model_calls": 1} + + +def test_a_reported_usages_cache_write_split_counts_only_where_it_holds_counts(client): + task_id = _send(client, "go") + for details in ({"cache_creation": {"ephemeral_1h_input_tokens": None, "ephemeral_5m_input_tokens": 7}}, + {"cache_creation": "x", "cache_write_tokens_details": [1, 2]}, + {"cache_creation": {"ephemeral_5m_input_tokens": "3", "ephemeral_1h_input_tokens": True, + "other": -1}}): + usage = _reported(Usage(input_tokens=1, output_tokens=1, provider_details=details)) + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + task = _settled(client, task_id) + assert task["status"]["state"] == "completed" + assert _usage(task)["provider_details"] == {"model_calls": 3, "cache_creation": {"ephemeral_5m_input_tokens": 7}} + + +def test_a_turn_sums_each_providers_cache_write_split_and_counts_usage_it_couldnt_read(client): + task_id = _send(client, "go") + for usage in ({"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 7813, + "cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 0}}, + {"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 20, + "cache_creation": {"ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 20}}, + {"inputTokens": 5, "outputTokens": 2}): + assert _proxy(client, "record", task_id=task_id, message="Thinking.", usage=usage).status_code == 200 + _proxy(client, "reply", task_id=task_id, parts=[{"kind": "text", "text": "Done."}]) + + assert _usage(_settled(client, task_id))["provider_details"] == { + "model_calls": 3, "unrecognized": 1, + "unreported": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 1, "cache_write_tokens": 1}, + "cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 20}} + + def test_an_env_tool_named_bash_is_held_to_all_its_recorded_arguments(terminal, monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) with TestClient(ProxyAgent(shell=False, long_poll_seconds=0.2).create_app()) as client: diff --git a/packages/agentenv-protocol/tests/test_usage.py b/packages/agentenv-protocol/tests/test_usage.py new file mode 100644 index 000000000..f9f3553b4 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_usage.py @@ -0,0 +1,140 @@ +"""``Usage``'s cache counts and includes-cache flag: validated like its other fields, written by ``to_dict`` only when +set, read back by ``from_dict`` from any protocol version, and carried by a failed result as by a successful one.""" + +import pytest + +from agentenv_protocol.a2a_agent.tasks.v1 import TaskResult, Usage + +FULL = Usage(tool_call_count=2, input_tokens=120, output_tokens=7, total_tokens=127, cache_read_tokens=100, + cache_write_tokens=20, input_includes_cache=True, cost_usd=0.01, provider_details={"model": "m"}) + + +@pytest.mark.parametrize("usage", [FULL, Usage(), Usage(input_tokens=5), Usage(input_includes_cache=False), + Usage(cache_read_tokens=0, cache_write_tokens=0)], + ids=["full", "empty", "no-cache-fields", "flag-false", "zero-cache"]) +def test_usage_round_trips_with_and_without_the_cache_fields(usage): + assert Usage.from_dict(usage.to_dict()) == usage + + +def test_to_dict_writes_the_cache_fields_only_when_set(): + assert FULL.to_dict() == {"tool_call_count": 2, "input_tokens": 120, "output_tokens": 7, "total_tokens": 127, + "cache_read_tokens": 100, "cache_write_tokens": 20, "input_includes_cache": True, + "cost_usd": 0.01, "provider_details": {"model": "m"}} + assert Usage(input_tokens=5, input_includes_cache=False).to_dict() == {"input_tokens": 5, + "input_includes_cache": False} + + +def test_an_older_agents_usage_reads_with_no_cache_fields_and_a_newer_ones_unknown_keys_are_ignored(): + older = Usage.from_dict({"tool_call_count": 1, "input_tokens": 9, "output_tokens": 4, + "provider_details": {"cache_read_tokens": 3}}) + assert (older.cache_read_tokens, older.cache_write_tokens, older.input_includes_cache) == (None, None, None) + assert older.provider_details == {"cache_read_tokens": 3} + assert Usage.from_dict({"input_tokens": 9, "reasoning_tokens": 2, "cache_read_tokens": None}) == Usage( + input_tokens=9) + + +@pytest.mark.parametrize("field", ["cache_read_tokens", "cache_write_tokens"]) +@pytest.mark.parametrize("value", [-1, 1.5, "3", True]) +def test_a_cache_count_that_isnt_a_non_negative_integer_is_refused(field, value): + with pytest.raises(ValueError, match=f"{field} must be a non-negative integer"): + Usage(**{field: value}) + with pytest.raises(ValueError, match=field): + Usage.from_dict({field: value}) + + +@pytest.mark.parametrize("value", [1, 0, "true", "false"]) +def test_an_includes_cache_flag_that_isnt_a_bool_is_refused(value): + with pytest.raises(ValueError, match="input_includes_cache must be true, false or null"): + Usage(input_includes_cache=value) + + +def test_from_dict_refuses_what_isnt_a_mapping(): + with pytest.raises(TypeError, match="usage must be a mapping"): + Usage.from_dict([("input_tokens", 1)]) + + +@pytest.mark.parametrize("details", [[], False, 0, "", "x", [("a", 1)]]) +def test_from_dict_refuses_provider_details_that_arent_a_mapping_even_when_empty(details): + with pytest.raises(TypeError, match="provider_details must be a mapping"): + Usage.from_dict({"input_tokens": 1, "provider_details": details}) + assert Usage.from_dict({"input_tokens": 1, "provider_details": None}) == Usage(input_tokens=1) + + +@pytest.mark.parametrize("usage", [Usage(cache_read_tokens=0), Usage(cache_write_tokens=3), + Usage(input_includes_cache=False)], ids=["read", "write", "flag"]) +def test_a_usage_with_only_cache_fields_isnt_empty(usage): + assert not usage.is_empty + assert Usage().is_empty + + +def test_a_failed_result_carries_its_usage(): + result = TaskResult.failure("agent.boom", "it broke", usage=FULL) + assert (result.outcome, result.usage) == ("failed", FULL) + assert TaskResult.failure("agent.boom", "it broke").usage == Usage() + + +# Real responses recorded through a LiteLLM proxy: one Claude prompt of 7,818 tokens, 7,809 of them cacheable, and one +# GPT prompt, through Chat Completions, Anthropic's Messages and OpenAI's Responses; the LiteLLM client's shape is as +# LiteLLM 1.83's client reads Anthropic's, and the Responses read with a cache hit is OpenAI's documented shape. +RESPONSES = { + "litellm-proxy-claude-writes": ({"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14, + "cache_creation_input_tokens": 7809, "cache_read_input_tokens": 0}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, cache_read_tokens=0, + cache_write_tokens=7809, input_includes_cache=True)), + "litellm-proxy-claude-reads": ({"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14, + "cache_creation_input_tokens": 0, "cache_read_input_tokens": 7809}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, cache_read_tokens=7809, + cache_write_tokens=0, input_includes_cache=True)), + "anthropic-messages-reads": ({"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 7809, "cache_creation": {"ephemeral_5m_input_tokens": 0}}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, cache_read_tokens=7809, + cache_write_tokens=0, input_includes_cache=True, + provider_details={"cache_creation": {"ephemeral_5m_input_tokens": 0}})), + "anthropic-messages-writes-for-an-hour": ( + {"input_tokens": 9, "output_tokens": 5, "cache_creation_input_tokens": 7813, "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 0}, "speed": "standard"}, + Usage(input_tokens=7822, output_tokens=5, total_tokens=7827, cache_read_tokens=0, cache_write_tokens=7813, + input_includes_cache=True, + provider_details={"cache_creation": {"ephemeral_1h_input_tokens": 7813, "ephemeral_5m_input_tokens": 0}})), + "openai-responses-gpt": ( + {"input_tokens": 7823, "output_tokens": 3, "total_tokens": 7826, "cost": None, + "input_tokens_details": {"audio_tokens": None, "cached_tokens": 0, "cache_write_tokens": 0, "text_tokens": None, + "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 0}}, + "output_tokens_details": {"reasoning_tokens": 0, "text_tokens": None}}, + Usage(input_tokens=7823, output_tokens=3, total_tokens=7826, cache_read_tokens=0, cache_write_tokens=0, + input_includes_cache=True, + provider_details={"cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 0}})), + "openai-responses-claude": ( + {"input_tokens": 7819, "output_tokens": 5, "total_tokens": 7824, "cost": None, + "input_tokens_details": {"audio_tokens": None, "cached_tokens": 0, "text_tokens": 7819}, + "output_tokens_details": {"reasoning_tokens": 0, "text_tokens": 5}}, + Usage(input_tokens=7819, output_tokens=5, total_tokens=7824, cache_read_tokens=0, input_includes_cache=True)), + "openai-responses-reads": ({"input_tokens": 7823, "output_tokens": 3, "total_tokens": 7826, + "input_tokens_details": {"cached_tokens": 7680}}, + Usage(input_tokens=7823, output_tokens=3, total_tokens=7826, cache_read_tokens=7680, + input_includes_cache=True)), + "litellm-client-claude-reads": ({"prompt_tokens": 7818, "completion_tokens": 5, "total_tokens": 7823, + "prompt_tokens_details": {"cached_tokens": 7809}, + "cache_creation_input_tokens": 0, "cache_read_input_tokens": 7809}, + Usage(input_tokens=7818, output_tokens=5, total_tokens=7823, + cache_read_tokens=7809, cache_write_tokens=0, input_includes_cache=True)), + "openai-gpt-reads": ({"prompt_tokens": 7822, "completion_tokens": 2, "total_tokens": 7824, + "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 7680}}, + Usage(input_tokens=7822, output_tokens=2, total_tokens=7824, cache_read_tokens=7680, + input_includes_cache=True)), + "openai-no-details": ({"prompt_tokens": 9, "completion_tokens": 4}, + Usage(input_tokens=9, output_tokens=4, total_tokens=13, input_includes_cache=True)), + "details-below-the-cache": ({"prompt_tokens": 9, "completion_tokens": 1, "prompt_tokens_details": {"cached_tokens": 50}}, + Usage(input_tokens=59, output_tokens=1, total_tokens=60, cache_read_tokens=50, + input_includes_cache=True)), + "nothing": ({}, Usage()), + "unreadable": ({"prompt_tokens": "9", "completion_tokens": True, "prompt_tokens_details": "x"}, Usage()), + "bedrock-camel-case": ({"inputTokens": 5, "outputTokens": 2, "cacheReadInputTokens": 1}, + Usage(provider_details={"unrecognized": ["cacheReadInputTokens", "inputTokens", + "outputTokens"]})), +} + + +@pytest.mark.parametrize("response, expected", RESPONSES.values(), ids=RESPONSES.keys()) +def test_a_model_responses_usage_reads_with_input_counting_the_cache(response, expected): + assert Usage.from_model_response(response) == expected diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index a4643c02a..a25b5086d 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -52,6 +52,11 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn. Its parts become the task's status message, with `usage` as a data part; a failure adds `{error_type, error_code, error_message}`. A success needs at least one part, and `builder().failed(...)` adds no text part, so add one. +- **Usage:** when the runtime counts prompt-cache tokens, put them in `Usage`'s `cache_read_tokens` and + `cache_write_tokens`, not `provider_details`, and set `input_includes_cache` to whether `input_tokens` counts them + (`None` when it can't tell). Set it per call: through a gateway it depends on the model behind it. Where the runtime + hands back a model response's own usage, `Usage.from_model_response` reads it. Code that sets these fields needs a + `requirements.txt` protocol floor at the first release that has them. - **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text never sent. diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index 0de563ffb..b9b446789 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -8,11 +8,13 @@ import httpx from agentenv_protocol import EnvironmentTool, ToolResult from agentenv_protocol.a2a_agent import ( + REPORTED_USAGE_KEY, ProxyBashAction, ProxyBashResult, ProxySkill, ProxyTextPart, ProxyToolAction, + Usage, ) from agent_env.a2a_agent import protocol @@ -66,11 +68,14 @@ def tools(self, *, timeout: float = 600.0) -> TurnTools: return TurnTools(self, timeout) async def record_model_call(self, message: str | dict[str, Any], *, model: str | None = None, - usage: dict[str, Any] | None = None) -> None: + usage: Usage | dict[str, Any] | None = None) -> None: """Record one of the model's calls, once it has returned: an OpenAI chat message, an Anthropic message or a string. From then on each call to the turn's tools must be one the last recorded call made, and all must run before the next is recorded or the turn ends; else ``EpisodeError``. Calls to tools the turn doesn't offer - aren't checked.""" + aren't checked. ``usage`` is the call's: a ``Usage``, taken as given, from a caller that knows what its counts + hold, or the response's own usage, which the proxy reads with ``Usage.from_model_response``.""" + if isinstance(usage, Usage): + usage = {REPORTED_USAGE_KEY: usage.to_dict()} await self._checked(self._proxy.record(self._task_id, message, model, usage)) async def end(self, response: str | None = None) -> None: diff --git a/tst/integration/agent/proxy_usage_local_test.py b/tst/integration/agent/proxy_usage_local_test.py new file mode 100644 index 000000000..3a46ef5a0 --- /dev/null +++ b/tst/integration/agent/proxy_usage_local_test.py @@ -0,0 +1,73 @@ +"""A played agent's turn reaches the host with the usage of the model calls its trainer recorded: an episode's proxy, +served in this process, sums an OpenAI-shaped and an Anthropic-shaped response's usage and a ``Usage`` the trainer +reports itself into one ``Usage`` whose ``input_tokens`` counts the cache, and the host stores it with the agent's reply +in the run's conversation. Local stores, no Docker.""" + +import pytest +from agentenv_protocol.a2a_agent.tasks.v1 import Usage + +from agent_env.a2a_agent import conversation_store +from agent_env.artifact.store import reset_artifact_store +from agent_env.config import configure, reset_config +from agent_env.task import EpisodeSpec, Task + +pytestmark = [pytest.mark.integration] + +OPENAI = {"prompt_tokens": 120, "completion_tokens": 7, "total_tokens": 127, "prompt_tokens_details": {"cached_tokens": 100}} +ANTHROPIC = {"input_tokens": 3, "output_tokens": 9, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20} +REPORTED = Usage(input_tokens=4, output_tokens=2, total_tokens=6, cache_read_tokens=10, input_includes_cache=False, + cost_usd=0.02) + + +@pytest.fixture +def local_backends(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(tmp_path / "sandboxes")) + configure() + reset_artifact_store() + try: + yield + finally: + reset_config() + reset_artifact_store() + + +def _stored_usage(context_id: str) -> list[dict]: + """The usage of each agent message the run's conversation holds.""" + conversation = conversation_store.get_conversation(context_id) + return [part["data"]["usage"] for message in conversation["messages"] if message["role"] == "agent" + for part in message["parts"] if part.get("kind") == "data" and "usage" in (part.get("data") or {})] + + +@pytest.mark.asyncio +async def test_a_turns_recorded_model_calls_reach_the_host_as_the_turns_usage(local_backends): + task = Task.from_dict({"id": "usage", "steps": [ + {"id": "policy", "type": "deploy_agent", "agent_name": "policy", "env_ids": []}, + {"id": "play", "type": "prompt_agent", "agent_name": "policy", "prompt_id": "play", "prompt": "Say hi.", + "timeout_seconds": 60}, + ]}) + episode = await EpisodeSpec.from_task(task, trainer_agents=["policy"]).start() + try: + turn = await episode.next_turn() + await turn.record_model_call({"role": "assistant", "content": "Thinking."}, model="policy-model", + usage=OPENAI) + await turn.record_model_call({"role": "assistant", "content": "Still thinking."}, model="policy-model", + usage=ANTHROPIC) + await turn.record_model_call("Hi.", model="policy-model", usage=REPORTED) + await turn.end("Hi.") + await episode.result() + finally: + await episode.abandon("the test ended") + + record = Task.get_instance(episode.id) + assert record.status == "completed" + [play] = record.context["prompt_responses"] + assert play["response"] == "Hi." + [usage] = _stored_usage(play["a2a_context_id"]) + assert Usage.from_dict(usage) == Usage( + tool_call_count=0, input_tokens=120 + 123 + 14, output_tokens=18, total_tokens=127 + 132 + 16, + cache_read_tokens=210, cache_write_tokens=20, input_includes_cache=True, cost_usd=0.02, + provider_details={"model_calls": 3, "unreported": {"cache_write_tokens": 2, "cost_usd": 2}}) diff --git a/tst/unit/a2a_agent/protocol_test.py b/tst/unit/a2a_agent/protocol_test.py index 6f6f41c19..05ad0728f 100644 --- a/tst/unit/a2a_agent/protocol_test.py +++ b/tst/unit/a2a_agent/protocol_test.py @@ -29,6 +29,7 @@ AgentIdentity, TaskRequest, TaskResult, + TextPart, Usage, a2a_agent, enable, @@ -168,6 +169,27 @@ def test_sdk_success_round_trips_through_platform_parser(): assert parsed.tool_call_count == 3 +def test_a_terminal_message_with_cache_usage_parses_and_its_usage_reads_back_whole(): + usage = Usage(tool_call_count=2, input_tokens=120, output_tokens=7, cache_read_tokens=100, cache_write_tokens=20, + input_includes_cache=True) + message = _sdk_terminal_message(TaskResult.success(parts=(TextPart(text="done"),), usage=usage)) + + parsed = TerminalResponse.from_message(message) + + assert (parsed.response_text, parsed.tool_call_count) == ("done", 2) + [sent] = [part["data"]["usage"] for part in message["parts"] if part.get("kind") == "data"] + assert Usage.from_dict(sent) == usage + + +def test_a_failed_turn_sends_its_usage(): + usage = Usage(input_tokens=5, output_tokens=2, input_includes_cache=False) + message = _sdk_terminal_message(TaskResult.failure("agent.boom", "it broke", usage=usage)) + + [sent] = [part["data"]["usage"] for part in message["parts"] if part.get("kind") == "data"] + assert Usage.from_dict(sent) == usage + assert TerminalResponse.from_message(message).error_code == "agent.boom" + + def test_sdk_failure_round_trips_through_platform_parser(): result = TaskResult.failure( "provider_rate_limited",