Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added .pr/6741-after.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added .pr/6741-before.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ def _compose_conversation_info(
agent_state.get("acp_supports_runtime_model_switch", False)
)
return ConversationInfo(
**state.model_dump(mode="json"),
**state.model_dump(mode="json", context={"use_snapshot": True}),
title=stored.title,
metrics=stored.metrics,
created_at=stored.created_at,
Expand Down
8 changes: 5 additions & 3 deletions openhands-sdk/openhands/sdk/event/conversation_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,11 @@ def from_conversation_state(
Returns:
A ConversationStateUpdateEvent with serialized state data
"""
# Create a snapshot with all important state fields
# Use mode='json' to ensure proper serialization including SecretStr
state_snapshot = state.model_dump(mode="json", exclude_none=True)
# mode='json' serializes SecretStr; use_snapshot drops per-call metrics
# lists so the snapshot stays under the WebSocket frame limit.
state_snapshot = state.model_dump(
mode="json", exclude_none=True, context={"use_snapshot": True}
)

# Use a special key "full_state" to indicate this is a full snapshot
return cls(key=FULL_STATE_KEY, value=state_snapshot)
Expand Down
35 changes: 35 additions & 0 deletions tests/agent_server/test_conversation_info_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
ConversationExecutionStatus,
ConversationState,
)
from openhands.sdk.llm.utils.metrics import Metrics
from openhands.sdk.security.confirmation_policy import NeverConfirm
from openhands.sdk.workspace import LocalWorkspace

Expand Down Expand Up @@ -400,3 +401,37 @@ def test_supports_runtime_model_switch_defaults_false():
info = _compose_conversation_info(stored, state)

assert info.supports_runtime_model_switch is False


def test_conversation_info_stats_omit_per_call_metrics():
"""REST conversation info carries aggregates, not per-call metrics (#6741)."""
agent = Agent(
llm=LLM(model="gpt-4o", api_key=SecretStr("test-key"), usage_id="test-llm"),
tools=[],
)
state = _make_state(agent)
metrics = Metrics(model_name="gpt-4o")
for i in range(50):
metrics.add_cost(0.01)
metrics.add_token_usage(
prompt_tokens=100,
completion_tokens=50,
cache_read_tokens=0,
cache_write_tokens=0,
context_window=8000,
response_id=f"resp{i}",
)
metrics.add_response_latency(1.5, f"resp{i}")
state.stats.usage_to_metrics["test-llm"] = metrics

info = _compose_conversation_info(_make_stored(state), state)

served = info.model_dump(mode="json")["stats"]["usage_to_metrics"]["test-llm"]
assert served["costs"] == []
assert served["response_latencies"] == []
assert served["token_usages"] == []
assert served["accumulated_cost"] == pytest.approx(0.5)
assert served["accumulated_token_usage"]["prompt_tokens"] == 5000
assert served["accumulated_token_usage"]["completion_tokens"] == 2500
# The live state, which is what gets persisted, is untouched.
assert len(state.stats.usage_to_metrics["test-llm"].costs) == 50
74 changes: 74 additions & 0 deletions tests/cross/test_remote_conversation_live_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@
from websockets.sync.client import connect

from openhands.agent_server.config import Config, load_config
from openhands.agent_server.session_protocol import MAX_FRAME_BYTES
from openhands.sdk import LLM, Agent, AgentContext, Conversation, Message, TextContent
from openhands.sdk.conversation import RemoteConversation
from openhands.sdk.conversation.conversation_stats import ConversationStats
from openhands.sdk.conversation.event_store import EventLog
from openhands.sdk.conversation.exceptions import WebSocketConnectionError
from openhands.sdk.event import (
Expand All @@ -56,6 +58,7 @@
from openhands.sdk.hooks import HookConfig, HookDefinition, HookMatcher
from openhands.sdk.io.local import LocalFileStore
from openhands.sdk.llm import MessageToolCall
from openhands.sdk.llm.utils.metrics import Metrics
from openhands.sdk.security.llm_analyzer import LLMSecurityAnalyzer
from openhands.sdk.skills import Skill
from openhands.sdk.subagent import AgentDefinition
Expand Down Expand Up @@ -2015,6 +2018,77 @@ async def fake_acompletion(self, messages, tools=None, **kwargs): # type: ignor
conv.close()


def _metrics_exceeding_frame_limit() -> Metrics:
metrics = Metrics(model_name="gpt-4o-mini")
calls = 0
while len(metrics.model_dump_json()) <= MAX_FRAME_BYTES:
for _ in range(500):
response_id = f"chatcmpl-{calls:06d}-{uuid4().hex}"
metrics.add_cost(0.0123)
metrics.add_token_usage(
prompt_tokens=120_000,
completion_tokens=512,
cache_read_tokens=110_000,
cache_write_tokens=2_048,
context_window=200_000,
response_id=response_id,
reasoning_tokens=256,
)
metrics.add_response_latency(3.21, response_id)
calls += 1
return metrics


def test_long_conversation_full_state_fits_session_frame(server_env):
"""Per-call metrics over 4 MiB must not break reconnects (#6741)."""
ws_host = server_env["host"].replace("http://", "ws://")
with httpx.Client(base_url=server_env["host"]) as client:
conversation_id = _start_idle_conversation(client)
event_service = server_env["conversation_service"]._event_services[
UUID(conversation_id)
]
state = event_service.get_conversation()._state
metrics = _metrics_exceeding_frame_limit()
with state:
state.stats = ConversationStats(usage_to_metrics={"agent": metrics})
assert len(state.model_dump_json()) > MAX_FRAME_BYTES

full_state = None
with connect(
f"{ws_host}/sockets/session/{conversation_id}", max_size=None
) as ws:
while full_state is None:
frame = json.loads(ws.recv(timeout=10))
event = frame.get("event") or {}
if event.get("key") == "full_state":
full_state = event["value"]

served = full_state["stats"]["usage_to_metrics"]["agent"]
assert "costs" not in served
assert "response_latencies" not in served
assert "token_usages" not in served
assert served["accumulated_cost"] == pytest.approx(metrics.accumulated_cost)
assert metrics.accumulated_token_usage is not None
assert (
served["accumulated_token_usage"]
== metrics.accumulated_token_usage.model_dump()
)

for response in (
client.get(f"/api/conversations/{conversation_id}"),
client.get("/api/conversations/search"),
):
response.raise_for_status()
assert len(response.content) < 100_000
# Persistence still keeps every call.
persisted = json.loads(
(Path(event_service.conversation_dir) / "base_state.json").read_text()
)
assert len(persisted["stats"]["usage_to_metrics"]["agent"]["costs"]) == len(
metrics.costs
)


def test_ask_agent_usage_is_in_conversation_info_at_once(server_env, patched_llm):
"""GET /api/conversations/{id} counts a side question's usage as soon as
ask_agent returns, with no run or other state change after it.
Expand Down
53 changes: 53 additions & 0 deletions tests/sdk/conversation/test_stats_update_event_snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,3 +192,56 @@ def test_stats_model_dump_with_snapshot_context_excludes_history():
assert "accumulated_token_usage" in metrics_data
assert metrics_data["accumulated_token_usage"]["prompt_tokens"] == 500
assert metrics_data["accumulated_token_usage"]["completion_tokens"] == 250


def _metrics_with_history(calls: int) -> Metrics:
metrics = Metrics(model_name="gpt-4")
for i in range(calls):
metrics.add_cost(0.01)
metrics.add_token_usage(
prompt_tokens=100,
completion_tokens=50,
cache_read_tokens=0,
cache_write_tokens=0,
context_window=8000,
response_id=f"resp{i}",
)
metrics.add_response_latency(1.5, f"resp{i}")
return metrics


def test_full_state_event_uses_snapshot_stats(state):
"""The full_state snapshot sent on (re)connect omits per-call metrics."""
state.stats.usage_to_metrics["default"] = _metrics_with_history(20)

event = ConversationStateUpdateEvent.from_conversation_state(state)

assert event.key == "full_state"
metrics_data = event.value["stats"]["usage_to_metrics"]["default"]
assert set(metrics_data) == {
"model_name",
"accumulated_cost",
"max_budget_per_task",
"accumulated_token_usage",
}
live = state.stats.usage_to_metrics["default"]
assert metrics_data["accumulated_cost"] == pytest.approx(live.accumulated_cost)
assert live.accumulated_token_usage is not None
assert (
metrics_data["accumulated_token_usage"]
== live.accumulated_token_usage.model_dump()
)
# Persistence keeps the full per-call history.
persisted = state.model_dump(mode="json")["stats"]["usage_to_metrics"]["default"]
assert len(persisted["costs"]) == 20
assert len(persisted["token_usages"]) == 20


def test_full_state_event_size_does_not_grow_with_llm_calls(state):
state.stats.usage_to_metrics["default"] = _metrics_with_history(1)
small = ConversationStateUpdateEvent.from_conversation_state(state)

state.stats.usage_to_metrics["default"] = _metrics_with_history(500)
large = ConversationStateUpdateEvent.from_conversation_state(state)

assert len(large.model_dump_json()) - len(small.model_dump_json()) < 100
Loading