Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions src/agent_env/agents/agents/browser-use/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,14 @@ async def run(self, request: TaskRequest[BrowserUseConfig]) -> TaskResult:
input_tokens=usage["input"],
output_tokens=usage["output"],
total_tokens=usage["input"] + usage["output"],
# browser-use counts the cache in its input on both of its routes: Anthropic's Messages, whose cache
# it adds in, and OpenAI's Chat Completions, whose prompt already holds it.
cache_read_tokens=usage["cache_read"],
cache_write_tokens=usage["cache_write"],
input_includes_cache=True,
cost_usd=result.get("cost_usd"),
provider_details={"steps": result["steps"], "model": result["model"], "requests": usage["requests"],
"cache_read_tokens": usage["cache_read"], "cache_write_tokens": usage["cache_write"],
"cost_usd": result.get("cost_usd"), "success": result.get("success")},
"success": result.get("success")},
))
.build()
)
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/browser-use/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
25 changes: 19 additions & 6 deletions src/agent_env/agents/agents/hermes/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -519,21 +519,34 @@ async def swap() -> None:


def _usage(result) -> Usage:
"""The turn's tool calls and the token counts and cost Hermes reported for its main model calls."""
"""The turn's tool calls and the token counts and cost Hermes reported for its main model calls, with
``input_tokens`` counting the cache: Hermes' ``input_tokens`` is the prompt less its cache reads and writes, so
they're added back. Calls that reported no counts at all got none from their endpoint, as streamed Claude answers
through a LiteLLM proxy, so the turn's counts are unknown rather than nothing."""
report = result.usage or {}

def count(name: str) -> int | None:
value = report.get(name)
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None

cost = report.get("estimated_cost_usd")
details = {key: report[key] for key in ("api_calls", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens",
"cost_status", "turn_exit_reason") if report.get(key) is not None}
details = {key: report[key] for key in ("api_calls", "reasoning_tokens", "cost_status", "turn_exit_reason")
if report.get(key) is not None}
counts = {name: count(name) for name in ("input_tokens", "output_tokens", "total_tokens", "cache_read_tokens",
"cache_write_tokens")}
if (count("api_calls") or 0) > 0 and not any(counts.values()):
return Usage(tool_call_count=result.tool_call_count,
provider_details={**details, "unreported_calls": details["api_calls"]})
read, write = counts["cache_read_tokens"], counts["cache_write_tokens"]
input_tokens = counts["input_tokens"]
return Usage(
tool_call_count=result.tool_call_count,
input_tokens=count("input_tokens"),
output_tokens=count("output_tokens"),
total_tokens=count("total_tokens"),
input_tokens=None if input_tokens is None else input_tokens + (read or 0) + (write or 0),
output_tokens=counts["output_tokens"],
total_tokens=counts["total_tokens"],
cache_read_tokens=read,
cache_write_tokens=write,
input_includes_cache=None if input_tokens is None else True,
cost_usd=cost if isinstance(cost, (int, float)) and not isinstance(cost, bool) and cost >= 0 else None,
provider_details=details,
)
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/hermes/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
jsonschema>=4.18,<5
38 changes: 37 additions & 1 deletion src/agent_env/agents/agents/openhands/openhands_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import uuid
from collections.abc import AsyncIterator, Callable, Mapping
from contextlib import asynccontextmanager, suppress
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Final, Literal, Protocol, Self
from urllib.parse import urlsplit
Expand Down Expand Up @@ -157,6 +157,42 @@ class Outcome:
error_type: Literal["agent_error", "infra_error"] = "agent_error"


def turn_usage(calls: list[tuple[Any, bool]], *, responses: int, tool_calls: int, cost: float | None) -> Usage:
"""A turn's usage from the ``TokenUsage`` OpenHands recorded for each model call, beside whether its LLM speaks
OpenAI's Responses API, with ``input_tokens`` counting the cache. A Responses call's input counts it. A Chat
Completions call's may or may not: LiteLLM reads cached tokens from OpenAI's ``prompt_tokens_details``, whose
prompt counts them, and from Anthropic's keys beside a prompt that doesn't, as a LiteLLM proxy sends Claude's.
The cache is added when the prompt is smaller than it, and the turn's flag is left unknown when the counts can't
say. OpenHands records no ``TokenUsage`` for a response without tokens, so the turn's ``responses`` beyond its
calls are counted in ``unreported_calls``."""
counted = [_counted(usage.prompt_tokens, usage.completion_tokens, usage.cache_read_tokens,
usage.cache_write_tokens, True if responses_api else None) for usage, responses_api in calls]
unreported = responses - len(calls)
details = {"reasoning_tokens": sum(usage.reasoning_tokens or 0 for usage, _ in calls),
**({"unreported_calls": unreported} if unreported else {})}
return replace(_summed(counted, tool_calls, details), cost_usd=cost)


def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]:
"""A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the
cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0."""
if counts_cache or read + write == 0:
return prompt, output, read, write, 1
if prompt < read + write:
return prompt + read + write, output, read, write, 1
return prompt, output, read, write, 0


def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage:
"""The calls' counts summed: none at all when no call reported usage, the flag unknown when a call's can't be."""
if not calls:
return Usage(tool_call_count=tool_calls, provider_details=details)
input_tokens, output_tokens, read, write, settled = (sum(column) for column in zip(*calls))
return Usage(tool_call_count=tool_calls, input_tokens=input_tokens, output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens, cache_read_tokens=read, cache_write_tokens=write,
input_includes_cache=True if settled == len(calls) else None, provider_details=details)


@dataclass(slots=True)
class _ConversationLock:
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
Expand Down
33 changes: 8 additions & 25 deletions src/agent_env/agents/agents/openhands/openhands_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,13 @@
from openhands.sdk.event import ActionEvent, Event
from openhands.sdk.event.conversation_error import ConversationErrorEvent
from openhands.sdk.event.error_classification import FailureKind
from openhands.sdk.llm.utils.metrics import Metrics
from openhands.sdk.mcp import MCPServer
from openhands.sdk.skills.skill import load_skills_from_dir
from openhands.tools.preset.default import get_default_condenser, get_default_tools
from pydantic import PrivateAttr

from conversation_run import ConversationRun
from openhands_agent import Outcome, Turn
from openhands_agent import Outcome, Turn, turn_usage

# The AgentEnv endpoint is a LiteLLM gateway; this provider prefix sends LiteLLM's requests to it as is.
_PROVIDER: Final[str] = "litellm_proxy"
Expand Down Expand Up @@ -88,29 +87,13 @@ def _agent(turn: Turn) -> Agent:


def _usage(agent: Agent, tool_calls: int) -> Usage:
"""What the turn's LLMs have spent: they are made for the turn, so they start at nothing."""
llms = (agent.llm, getattr(agent.condenser, "llm", None))
metrics = Metrics()
for each in {id(llm.metrics): llm.metrics for llm in llms if llm is not None}.values():
metrics.merge(each)

def spent(name: str) -> int:
return getattr(metrics.accumulated_token_usage, name, 0) or 0

input_tokens, output_tokens = spent("prompt_tokens"), spent("completion_tokens")
cost = metrics.accumulated_cost
return Usage(
tool_call_count=tool_calls,
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens,
cost_usd=cost or None,
provider_details={
"cache_read_tokens": spent("cache_read_tokens"),
"cache_write_tokens": spent("cache_write_tokens"),
"reasoning_tokens": spent("reasoning_tokens"),
},
)
"""What the turn's LLMs have spent, call by call: they are made for the turn, so they start at nothing. Each
response they got has a latency recorded, with or without usage."""
llms = {id(llm.metrics): llm for llm in (agent.llm, getattr(agent.condenser, "llm", None)) if llm is not None}
calls = [(usage, llm.uses_responses_api()) for llm in llms.values() for usage in llm.metrics.token_usages]
responses = sum(len(llm.metrics.response_latencies) for llm in llms.values())
cost = sum(llm.metrics.accumulated_cost for llm in llms.values())
return turn_usage(calls, responses=responses, tool_calls=tool_calls, cost=cost or None)


def _unstarted(exc: Exception) -> Outcome:
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/openhands/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
openhands-sdk==1.58.2
openhands-tools==1.58.2
23 changes: 19 additions & 4 deletions src/agent_env/agents/agents/pi/pi_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,20 +255,35 @@ def _record(run: _Run, event: dict[str, Any]) -> None:


def _usage(run: _Run) -> Usage:
"""The turn's usage, summed over its assistant messages, with ``input_tokens`` counting the cache: pi's ``input``
is the prompt less its cache reads and writes, so they're added back. A message whose usage is all zero got none
from its endpoint, as a streamed Claude answer through a LiteLLM proxy, and is counted in ``unreported_calls``
instead of as nothing used."""
totals = {"input": 0, "output": 0, "totalTokens": 0, "cacheRead": 0, "cacheWrite": 0}
cost = 0.0
cost, reported, unreported = 0.0, 0, 0
for message in run.assistant_messages:
usage = message.get("usage") or {}
counts = {key: int(usage.get(key) or 0) for key in totals}
if not any(counts.values()):
unreported += 1
continue
reported += 1
for key in totals:
totals[key] += int(usage.get(key) or 0)
totals[key] += counts[key]
cost += float((usage.get("cost") or {}).get("total") or 0)
details = {"unreported_calls": unreported} if unreported else {}
if not reported:
return Usage(tool_call_count=run.tool_calls, provider_details=details)
return Usage(
tool_call_count=run.tool_calls,
input_tokens=totals["input"],
input_tokens=totals["input"] + totals["cacheRead"] + totals["cacheWrite"],
output_tokens=totals["output"],
total_tokens=totals["totalTokens"],
cache_read_tokens=totals["cacheRead"],
cache_write_tokens=totals["cacheWrite"],
input_includes_cache=True,
cost_usd=cost or None,
provider_details={"cache_read_tokens": totals["cacheRead"], "cache_write_tokens": totals["cacheWrite"]},
provider_details=details,
)


Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/pi/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose Usage takes the prompt cache's
# counts (cache_read_tokens, cache_write_tokens, input_includes_cache); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/proxy/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that takes the native trajectory format
# the agent declares (enable(TRAJECTORY_V1, native=...)); raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.588
# image is built again only when this folder changes. The floor is the first whose proxy reports each turn's model usage,
# as the driver records it; raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.597
46 changes: 45 additions & 1 deletion src/agent_env/agents/agents/pydantic-ai/pydantic_ai_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import tarfile
import tempfile
import uuid
from collections.abc import AsyncIterator, Callable, Mapping
from collections.abc import AsyncIterator, Callable, Iterable, Mapping
from contextlib import asynccontextmanager, suppress
from dataclasses import dataclass, field
from pathlib import Path
Expand Down Expand Up @@ -165,6 +165,50 @@ class Outcome:
error_type: Literal["agent_error", "infra_error"] = "agent_error"


def turn_usage(responses: Iterable[Any], *, tool_calls: int, requests: int) -> Usage:
"""A turn's usage from the usage Pydantic AI kept of each model response (its ``RequestUsage``), with
``input_tokens`` counting the cache. Pydantic AI reads its cache counts from OpenAI's ``prompt_tokens_details``,
whose prompt counts them; a LiteLLM proxy's Claude answers send Anthropic's ``cache_read_input_tokens`` and
``cache_creation_input_tokens`` beside a prompt that doesn't, which Pydantic AI keeps in ``details``. Those are
added when the prompt is smaller than them, and leave the turn's flag unknown otherwise, since the counts can't say.
A response that kept no usage at all, as a streamed Claude answer through that proxy, is counted in
``unreported_calls`` instead of as nothing used."""
calls, unreported = [], 0
for usage in responses:
if not (usage.input_tokens or usage.output_tokens):
unreported += 1
continue
details = usage.details or {}
if usage.cache_read_tokens or usage.cache_write_tokens:
read, write, counts_cache = usage.cache_read_tokens, usage.cache_write_tokens, True
else:
read = details.get("cache_read_input_tokens", 0)
write = details.get("cache_creation_input_tokens", 0)
counts_cache = None
calls.append(_counted(usage.input_tokens, usage.output_tokens, read, write, counts_cache))
return _summed(calls, tool_calls, {"requests": requests, **({"unreported_calls": unreported} if unreported else {})})


def _counted(prompt: int, output: int, read: int, write: int, counts_cache: bool | None) -> tuple[int, ...]:
"""A call's input counting its cache, its output, cache read, cache write, and 1 when whether its prompt held the
cache is known (said so, ``counts_cache``; nothing cached; or a prompt smaller than the cache), else 0."""
if counts_cache or read + write == 0:
return prompt, output, read, write, 1
if prompt < read + write:
return prompt + read + write, output, read, write, 1
return prompt, output, read, write, 0


def _summed(calls: list[tuple[int, ...]], tool_calls: int, details: dict[str, Any]) -> Usage:
"""The calls' counts summed: none at all when no call reported usage, the flag unknown when a call's can't be."""
if not calls:
return Usage(tool_call_count=tool_calls, provider_details=details)
input_tokens, output_tokens, read, write, settled = (sum(column) for column in zip(*calls))
return Usage(tool_call_count=tool_calls, input_tokens=input_tokens, output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens, cache_read_tokens=read, cache_write_tokens=write,
input_includes_cache=True if settled == len(calls) else None, provider_details=details)


@dataclass(slots=True)
class _ConversationLock:
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
Expand Down
Loading
Loading