diff --git a/.pr/6741-after.png b/.pr/6741-after.png new file mode 100644 index 00000000000..2d158b9fca2 Binary files /dev/null and b/.pr/6741-after.png differ diff --git a/.pr/6741-before.png b/.pr/6741-before.png new file mode 100644 index 00000000000..0ff23773c17 Binary files /dev/null and b/.pr/6741-before.png differ diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index ef26ee84abd..4c7d9849dab 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -418,7 +418,7 @@ def _compose_conversation_info( agent_state.get("acp_supports_runtime_model_switch", False) ) return ConversationInfo( - **state.model_dump(mode="json"), + **state.model_dump(mode="json", context={"use_snapshot": True}), title=stored.title, metrics=stored.metrics, created_at=stored.created_at, diff --git a/openhands-sdk/openhands/sdk/event/conversation_state.py b/openhands-sdk/openhands/sdk/event/conversation_state.py index 33f5b7a01ec..caa968cb107 100644 --- a/openhands-sdk/openhands/sdk/event/conversation_state.py +++ b/openhands-sdk/openhands/sdk/event/conversation_state.py @@ -93,9 +93,11 @@ def from_conversation_state( Returns: A ConversationStateUpdateEvent with serialized state data """ - # Create a snapshot with all important state fields - # Use mode='json' to ensure proper serialization including SecretStr - state_snapshot = state.model_dump(mode="json", exclude_none=True) + # mode='json' serializes SecretStr; use_snapshot drops per-call metrics + # lists so the snapshot stays under the WebSocket frame limit. + state_snapshot = state.model_dump( + mode="json", exclude_none=True, context={"use_snapshot": True} + ) # Use a special key "full_state" to indicate this is a full snapshot return cls(key=FULL_STATE_KEY, value=state_snapshot) diff --git a/tests/agent_server/test_conversation_info_model.py b/tests/agent_server/test_conversation_info_model.py index 1fcd5a6e7ec..adeb2b9acae 100644 --- a/tests/agent_server/test_conversation_info_model.py +++ b/tests/agent_server/test_conversation_info_model.py @@ -34,6 +34,7 @@ ConversationExecutionStatus, ConversationState, ) +from openhands.sdk.llm.utils.metrics import Metrics from openhands.sdk.security.confirmation_policy import NeverConfirm from openhands.sdk.workspace import LocalWorkspace @@ -400,3 +401,37 @@ def test_supports_runtime_model_switch_defaults_false(): info = _compose_conversation_info(stored, state) assert info.supports_runtime_model_switch is False + + +def test_conversation_info_stats_omit_per_call_metrics(): + """REST conversation info carries aggregates, not per-call metrics (#6741).""" + agent = Agent( + llm=LLM(model="gpt-4o", api_key=SecretStr("test-key"), usage_id="test-llm"), + tools=[], + ) + state = _make_state(agent) + metrics = Metrics(model_name="gpt-4o") + for i in range(50): + metrics.add_cost(0.01) + metrics.add_token_usage( + prompt_tokens=100, + completion_tokens=50, + cache_read_tokens=0, + cache_write_tokens=0, + context_window=8000, + response_id=f"resp{i}", + ) + metrics.add_response_latency(1.5, f"resp{i}") + state.stats.usage_to_metrics["test-llm"] = metrics + + info = _compose_conversation_info(_make_stored(state), state) + + served = info.model_dump(mode="json")["stats"]["usage_to_metrics"]["test-llm"] + assert served["costs"] == [] + assert served["response_latencies"] == [] + assert served["token_usages"] == [] + assert served["accumulated_cost"] == pytest.approx(0.5) + assert served["accumulated_token_usage"]["prompt_tokens"] == 5000 + assert served["accumulated_token_usage"]["completion_tokens"] == 2500 + # The live state, which is what gets persisted, is untouched. + assert len(state.stats.usage_to_metrics["test-llm"].costs) == 50 diff --git a/tests/cross/test_remote_conversation_live_server.py b/tests/cross/test_remote_conversation_live_server.py index 0d38d7bd73b..972fc1d765d 100644 --- a/tests/cross/test_remote_conversation_live_server.py +++ b/tests/cross/test_remote_conversation_live_server.py @@ -32,8 +32,10 @@ from websockets.sync.client import connect from openhands.agent_server.config import Config, load_config +from openhands.agent_server.session_protocol import MAX_FRAME_BYTES from openhands.sdk import LLM, Agent, AgentContext, Conversation, Message, TextContent from openhands.sdk.conversation import RemoteConversation +from openhands.sdk.conversation.conversation_stats import ConversationStats from openhands.sdk.conversation.event_store import EventLog from openhands.sdk.conversation.exceptions import WebSocketConnectionError from openhands.sdk.event import ( @@ -56,6 +58,7 @@ from openhands.sdk.hooks import HookConfig, HookDefinition, HookMatcher from openhands.sdk.io.local import LocalFileStore from openhands.sdk.llm import MessageToolCall +from openhands.sdk.llm.utils.metrics import Metrics from openhands.sdk.security.llm_analyzer import LLMSecurityAnalyzer from openhands.sdk.skills import Skill from openhands.sdk.subagent import AgentDefinition @@ -2015,6 +2018,77 @@ async def fake_acompletion(self, messages, tools=None, **kwargs): # type: ignor conv.close() +def _metrics_exceeding_frame_limit() -> Metrics: + metrics = Metrics(model_name="gpt-4o-mini") + calls = 0 + while len(metrics.model_dump_json()) <= MAX_FRAME_BYTES: + for _ in range(500): + response_id = f"chatcmpl-{calls:06d}-{uuid4().hex}" + metrics.add_cost(0.0123) + metrics.add_token_usage( + prompt_tokens=120_000, + completion_tokens=512, + cache_read_tokens=110_000, + cache_write_tokens=2_048, + context_window=200_000, + response_id=response_id, + reasoning_tokens=256, + ) + metrics.add_response_latency(3.21, response_id) + calls += 1 + return metrics + + +def test_long_conversation_full_state_fits_session_frame(server_env): + """Per-call metrics over 4 MiB must not break reconnects (#6741).""" + ws_host = server_env["host"].replace("http://", "ws://") + with httpx.Client(base_url=server_env["host"]) as client: + conversation_id = _start_idle_conversation(client) + event_service = server_env["conversation_service"]._event_services[ + UUID(conversation_id) + ] + state = event_service.get_conversation()._state + metrics = _metrics_exceeding_frame_limit() + with state: + state.stats = ConversationStats(usage_to_metrics={"agent": metrics}) + assert len(state.model_dump_json()) > MAX_FRAME_BYTES + + full_state = None + with connect( + f"{ws_host}/sockets/session/{conversation_id}", max_size=None + ) as ws: + while full_state is None: + frame = json.loads(ws.recv(timeout=10)) + event = frame.get("event") or {} + if event.get("key") == "full_state": + full_state = event["value"] + + served = full_state["stats"]["usage_to_metrics"]["agent"] + assert "costs" not in served + assert "response_latencies" not in served + assert "token_usages" not in served + assert served["accumulated_cost"] == pytest.approx(metrics.accumulated_cost) + assert metrics.accumulated_token_usage is not None + assert ( + served["accumulated_token_usage"] + == metrics.accumulated_token_usage.model_dump() + ) + + for response in ( + client.get(f"/api/conversations/{conversation_id}"), + client.get("/api/conversations/search"), + ): + response.raise_for_status() + assert len(response.content) < 100_000 + # Persistence still keeps every call. + persisted = json.loads( + (Path(event_service.conversation_dir) / "base_state.json").read_text() + ) + assert len(persisted["stats"]["usage_to_metrics"]["agent"]["costs"]) == len( + metrics.costs + ) + + def test_ask_agent_usage_is_in_conversation_info_at_once(server_env, patched_llm): """GET /api/conversations/{id} counts a side question's usage as soon as ask_agent returns, with no run or other state change after it. diff --git a/tests/sdk/conversation/test_stats_update_event_snapshot.py b/tests/sdk/conversation/test_stats_update_event_snapshot.py index a7e82c020cb..c8b3f09abec 100644 --- a/tests/sdk/conversation/test_stats_update_event_snapshot.py +++ b/tests/sdk/conversation/test_stats_update_event_snapshot.py @@ -192,3 +192,56 @@ def test_stats_model_dump_with_snapshot_context_excludes_history(): assert "accumulated_token_usage" in metrics_data assert metrics_data["accumulated_token_usage"]["prompt_tokens"] == 500 assert metrics_data["accumulated_token_usage"]["completion_tokens"] == 250 + + +def _metrics_with_history(calls: int) -> Metrics: + metrics = Metrics(model_name="gpt-4") + for i in range(calls): + metrics.add_cost(0.01) + metrics.add_token_usage( + prompt_tokens=100, + completion_tokens=50, + cache_read_tokens=0, + cache_write_tokens=0, + context_window=8000, + response_id=f"resp{i}", + ) + metrics.add_response_latency(1.5, f"resp{i}") + return metrics + + +def test_full_state_event_uses_snapshot_stats(state): + """The full_state snapshot sent on (re)connect omits per-call metrics.""" + state.stats.usage_to_metrics["default"] = _metrics_with_history(20) + + event = ConversationStateUpdateEvent.from_conversation_state(state) + + assert event.key == "full_state" + metrics_data = event.value["stats"]["usage_to_metrics"]["default"] + assert set(metrics_data) == { + "model_name", + "accumulated_cost", + "max_budget_per_task", + "accumulated_token_usage", + } + live = state.stats.usage_to_metrics["default"] + assert metrics_data["accumulated_cost"] == pytest.approx(live.accumulated_cost) + assert live.accumulated_token_usage is not None + assert ( + metrics_data["accumulated_token_usage"] + == live.accumulated_token_usage.model_dump() + ) + # Persistence keeps the full per-call history. + persisted = state.model_dump(mode="json")["stats"]["usage_to_metrics"]["default"] + assert len(persisted["costs"]) == 20 + assert len(persisted["token_usages"]) == 20 + + +def test_full_state_event_size_does_not_grow_with_llm_calls(state): + state.stats.usage_to_metrics["default"] = _metrics_with_history(1) + small = ConversationStateUpdateEvent.from_conversation_state(state) + + state.stats.usage_to_metrics["default"] = _metrics_with_history(500) + large = ConversationStateUpdateEvent.from_conversation_state(state) + + assert len(large.model_dump_json()) - len(small.model_dump_json()) < 100