diff --git a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py index 00e5db67426..59f2c639c63 100644 --- a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py +++ b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py @@ -3334,6 +3334,8 @@ def ask_agent(self, question: str) -> str: This bypasses the normal conversation flow and does **not** modify, persist, or become part of the conversation state. The request is not remembered by the main agent, no events are recorded, and execution status is untouched. + Only the usage of the call is kept: it is counted in the conversation's + stats under ``ask-agent-llm`` and saved with them. It is also thread-safe and may be called while `conversation.run()` is executing in another thread. @@ -3393,6 +3395,9 @@ def ask_agent(self, question: str) -> str: store=False, call_context=self._llm_call_context, ) + # The call recorded its usage inside ``state.stats``; nothing else + # changes here that would save it. + self._state.save_soon() message = response.message diff --git a/openhands-sdk/openhands/sdk/conversation/state.py b/openhands-sdk/openhands/sdk/conversation/state.py index e5388ce1fea..00ea92f2544 100644 --- a/openhands-sdk/openhands/sdk/conversation/state.py +++ b/openhands-sdk/openhands/sdk/conversation/state.py @@ -906,12 +906,38 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Context manager exit — flushes any deferred save.""" try: self._save_depth -= 1 - if self._save_depth == 0 and self._dirty: - fs = self._fs - autosave_enabled = self._autosave_enabled - if autosave_enabled and fs is not None: - self._save_base_state(fs) - self._dirty = False + self._flush_deferred_save() + finally: + self._lock.release() + + def _flush_deferred_save(self) -> None: + if self._save_depth > 0 or not self._dirty: + return + # Cleared before the save: a change that save_soon() marks from another + # thread while this one serializes stays pending for the next flush. + self._dirty = False + fs = self._fs + if self._autosave_enabled and fs is not None: + try: + self._save_base_state(fs) + except Exception: + self._dirty = True + raise + + def save_soon(self) -> None: + """Persist a change made inside a field, without waiting for the lock. + + Assigning a field autosaves; changing a field's value in place (usage + recorded in ``stats``) does not. With the lock free the state is saved + now. Otherwise the change is marked pending and the lock holder's + ``with`` block saves it on exit, so a caller that must not wait for a + running step (``ask_agent``) never does. + """ + self._dirty = True + if not self._lock.acquire(blocking=False): + return + try: + self._flush_deferred_save() finally: self._lock.release() diff --git a/tests/cross/test_remote_conversation_live_server.py b/tests/cross/test_remote_conversation_live_server.py index b662291af2a..8aa0ca1e81c 100644 --- a/tests/cross/test_remote_conversation_live_server.py +++ b/tests/cross/test_remote_conversation_live_server.py @@ -1916,6 +1916,29 @@ async def fake_acompletion(self, messages, tools=None, **kwargs): # type: ignor conv.close() +def test_ask_agent_usage_is_in_conversation_info_at_once(server_env, patched_llm): + """GET /api/conversations/{id} counts a side question's usage as soon as + ask_agent returns, with no run or other state change after it. + """ + agent = Agent(llm=LLM(model="gpt-4o-mini", api_key=SecretStr("test")), tools=[]) + workspace = RemoteWorkspace( + host=server_env["host"], working_dir="/tmp/workspace/project" + ) + conv: RemoteConversation = Conversation(agent=agent, workspace=workspace) + try: + conv.ask_agent("How's the progress?") + + with httpx.Client(base_url=server_env["host"]) as client: + response = client.get(f"/api/conversations/{conv.id}") + response.raise_for_status() + usage_to_metrics = response.json()["stats"]["usage_to_metrics"] + assert "ask-agent-llm" in usage_to_metrics + token_usage = usage_to_metrics["ask-agent-llm"]["accumulated_token_usage"] + assert token_usage["prompt_tokens"] == 7 # one patched_llm call + finally: + conv.close() + + def test_events_not_lost_during_client_disconnection( server_env, monkeypatch: pytest.MonkeyPatch ): diff --git a/tests/sdk/conversation/test_ask_agent.py b/tests/sdk/conversation/test_ask_agent.py index c976646b340..a7810e2ce7e 100644 --- a/tests/sdk/conversation/test_ask_agent.py +++ b/tests/sdk/conversation/test_ask_agent.py @@ -1,7 +1,9 @@ """Tests for ask_agent functionality in conversation classes.""" import json +import threading from collections.abc import Sequence +from pathlib import Path from unittest.mock import Mock, patch import pytest @@ -301,6 +303,87 @@ def transport(llm, *args, **kwargs): assert conv.llm_registry.get("ask-agent-llm").telemetry is not agent.llm.telemetry +def saved_prompt_tokens(conv, usage_id: str = "ask-agent-llm") -> int | None: + """Prompt tokens ``base_state.json`` holds for a usage id right now.""" + saved = json.loads( + (Path(conv.state.persistence_dir) / "base_state.json").read_text() + ) + metrics = saved["stats"]["usage_to_metrics"].get(usage_id) + return metrics["accumulated_token_usage"]["prompt_tokens"] if metrics else None + + +@patch("openhands.sdk.llm.llm.LLM._transport_call", autospec=True) +def test_ask_agent_usage_is_saved_with_the_conversation(mock_transport, tmp_path): + """The usage of a side question is on disk when the call returns and is + still counted by a conversation resumed from that state. + """ + mock_transport.return_value = create_mock_model_response("answer") + conv = Conversation( + agent=create_test_agent(), + persistence_dir=str(tmp_path / "persist"), + workspace=str(tmp_path), + ) + + conv.ask_agent("How's the progress?") + + assert saved_prompt_tokens(conv) == 10 + + resumed = Conversation( + agent=create_test_agent(), + persistence_dir=str(tmp_path / "persist"), + workspace=str(tmp_path), + conversation_id=conv.id, + ) + usage = resumed.conversation_stats.usage_to_metrics[ + "ask-agent-llm" + ].accumulated_token_usage + assert usage is not None + assert usage.prompt_tokens == 10 + + resumed.ask_agent("And now?") + + assert saved_prompt_tokens(resumed) == 20 + + +@patch("openhands.sdk.llm.llm.LLM._transport_call", autospec=True) +def test_ask_agent_does_not_wait_for_the_state_lock_to_save_usage( + mock_transport, tmp_path +): + """While another thread holds the state lock (a running step), ask_agent + still answers at once; its usage is saved when that thread lets go. + """ + mock_transport.return_value = create_mock_model_response("answer") + conv = Conversation( + agent=create_test_agent(), + persistence_dir=str(tmp_path / "persist"), + workspace=str(tmp_path), + ) + conv.ask_agent("first") # initializes the agent, which takes the lock + locked = threading.Event() + release = threading.Event() + + def hold_state_lock() -> None: + with conv.state: + locked.set() + release.wait(timeout=30) + + holder = threading.Thread(target=hold_state_lock) + holder.start() + assert locked.wait(timeout=10) + answers: list[str] = [] + asker = threading.Thread(target=lambda: answers.append(conv.ask_agent("second"))) + asker.start() + asker.join(timeout=10) + answered_while_locked = not asker.is_alive() + release.set() + holder.join(timeout=10) + asker.join(timeout=10) + + assert answered_while_locked + assert answers == ["answer"] + assert saved_prompt_tokens(conv) == 20 + + @patch("openhands.sdk.conversation.impl.remote_conversation.WebSocketCallbackClient") def test_remote_conversation_ask_agent(mock_ws_client, agent): mock_ws_client.return_value.wait_until_ready.return_value = True