Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3334,6 +3334,8 @@ def ask_agent(self, question: str) -> str:
This bypasses the normal conversation flow and does **not** modify, persist,
or become part of the conversation state. The request is not remembered by
the main agent, no events are recorded, and execution status is untouched.
Only the usage of the call is kept: it is counted in the conversation's
stats under ``ask-agent-llm`` and saved with them.
It is also thread-safe and may be called while `conversation.run()` is
executing in another thread.

Expand Down Expand Up @@ -3393,6 +3395,9 @@ def ask_agent(self, question: str) -> str:
store=False,
call_context=self._llm_call_context,
)
# The call recorded its usage inside ``state.stats``; nothing else
# changes here that would save it.
self._state.save_soon()

message = response.message

Expand Down
38 changes: 32 additions & 6 deletions openhands-sdk/openhands/sdk/conversation/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -906,12 +906,38 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
"""Context manager exit — flushes any deferred save."""
try:
self._save_depth -= 1
if self._save_depth == 0 and self._dirty:
fs = self._fs
autosave_enabled = self._autosave_enabled
if autosave_enabled and fs is not None:
self._save_base_state(fs)
self._dirty = False
self._flush_deferred_save()
finally:
self._lock.release()

def _flush_deferred_save(self) -> None:
if self._save_depth > 0 or not self._dirty:
return
# Cleared before the save: a change that save_soon() marks from another
# thread while this one serializes stays pending for the next flush.
self._dirty = False
fs = self._fs
if self._autosave_enabled and fs is not None:
try:
self._save_base_state(fs)
except Exception:
self._dirty = True
raise

def save_soon(self) -> None:
"""Persist a change made inside a field, without waiting for the lock.

Assigning a field autosaves; changing a field's value in place (usage
recorded in ``stats``) does not. With the lock free the state is saved
now. Otherwise the change is marked pending and the lock holder's
``with`` block saves it on exit, so a caller that must not wait for a
running step (``ask_agent``) never does.
"""
self._dirty = True
if not self._lock.acquire(blocking=False):
return
try:
self._flush_deferred_save()
finally:
self._lock.release()

Expand Down
23 changes: 23 additions & 0 deletions tests/cross/test_remote_conversation_live_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -1916,6 +1916,29 @@ async def fake_acompletion(self, messages, tools=None, **kwargs): # type: ignor
conv.close()


def test_ask_agent_usage_is_in_conversation_info_at_once(server_env, patched_llm):
"""GET /api/conversations/{id} counts a side question's usage as soon as
ask_agent returns, with no run or other state change after it.
"""
agent = Agent(llm=LLM(model="gpt-4o-mini", api_key=SecretStr("test")), tools=[])
workspace = RemoteWorkspace(
host=server_env["host"], working_dir="/tmp/workspace/project"
)
conv: RemoteConversation = Conversation(agent=agent, workspace=workspace)
try:
conv.ask_agent("How's the progress?")

with httpx.Client(base_url=server_env["host"]) as client:
response = client.get(f"/api/conversations/{conv.id}")
response.raise_for_status()
usage_to_metrics = response.json()["stats"]["usage_to_metrics"]
assert "ask-agent-llm" in usage_to_metrics
token_usage = usage_to_metrics["ask-agent-llm"]["accumulated_token_usage"]
assert token_usage["prompt_tokens"] == 7 # one patched_llm call
finally:
conv.close()


def test_events_not_lost_during_client_disconnection(
server_env, monkeypatch: pytest.MonkeyPatch
):
Expand Down
83 changes: 83 additions & 0 deletions tests/sdk/conversation/test_ask_agent.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
"""Tests for ask_agent functionality in conversation classes."""

import json
import threading
from collections.abc import Sequence
from pathlib import Path
from unittest.mock import Mock, patch

import pytest
Expand Down Expand Up @@ -301,6 +303,87 @@ def transport(llm, *args, **kwargs):
assert conv.llm_registry.get("ask-agent-llm").telemetry is not agent.llm.telemetry


def saved_prompt_tokens(conv, usage_id: str = "ask-agent-llm") -> int | None:
"""Prompt tokens ``base_state.json`` holds for a usage id right now."""
saved = json.loads(
(Path(conv.state.persistence_dir) / "base_state.json").read_text()
)
metrics = saved["stats"]["usage_to_metrics"].get(usage_id)
return metrics["accumulated_token_usage"]["prompt_tokens"] if metrics else None


@patch("openhands.sdk.llm.llm.LLM._transport_call", autospec=True)
def test_ask_agent_usage_is_saved_with_the_conversation(mock_transport, tmp_path):
"""The usage of a side question is on disk when the call returns and is
still counted by a conversation resumed from that state.
"""
mock_transport.return_value = create_mock_model_response("answer")
conv = Conversation(
agent=create_test_agent(),
persistence_dir=str(tmp_path / "persist"),
workspace=str(tmp_path),
)

conv.ask_agent("How's the progress?")

assert saved_prompt_tokens(conv) == 10

resumed = Conversation(
agent=create_test_agent(),
persistence_dir=str(tmp_path / "persist"),
workspace=str(tmp_path),
conversation_id=conv.id,
)
usage = resumed.conversation_stats.usage_to_metrics[
"ask-agent-llm"
].accumulated_token_usage
assert usage is not None
assert usage.prompt_tokens == 10

resumed.ask_agent("And now?")

assert saved_prompt_tokens(resumed) == 20


@patch("openhands.sdk.llm.llm.LLM._transport_call", autospec=True)
def test_ask_agent_does_not_wait_for_the_state_lock_to_save_usage(
mock_transport, tmp_path
):
"""While another thread holds the state lock (a running step), ask_agent
still answers at once; its usage is saved when that thread lets go.
"""
mock_transport.return_value = create_mock_model_response("answer")
conv = Conversation(
agent=create_test_agent(),
persistence_dir=str(tmp_path / "persist"),
workspace=str(tmp_path),
)
conv.ask_agent("first") # initializes the agent, which takes the lock
locked = threading.Event()
release = threading.Event()

def hold_state_lock() -> None:
with conv.state:
locked.set()
release.wait(timeout=30)

holder = threading.Thread(target=hold_state_lock)
holder.start()
assert locked.wait(timeout=10)
answers: list[str] = []
asker = threading.Thread(target=lambda: answers.append(conv.ask_agent("second")))
asker.start()
asker.join(timeout=10)
answered_while_locked = not asker.is_alive()
release.set()
holder.join(timeout=10)
asker.join(timeout=10)

assert answered_while_locked
assert answers == ["answer"]
assert saved_prompt_tokens(conv) == 20


@patch("openhands.sdk.conversation.impl.remote_conversation.WebSocketCallbackClient")
def test_remote_conversation_ask_agent(mock_ws_client, agent):
mock_ws_client.return_value.wait_until_ready.return_value = True
Expand Down
Loading