From dd08bc33bc624f06045332765a575544033de994 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Fri, 6 Nov 2026 17:49:40 +0100 Subject: [PATCH 1/9] fix(agent-server): stop state-lock waiters from deadlocking tool execution arun() holds a conversation's state FIFOLock on the event-loop thread for a whole tool step, and the sequential tool path ran the tool on asyncio's shared default executor. Requests that wait on that lock (WebSocket subscribe snapshots, status reads, send_message, pause) also parked default-executor threads. Once waiters filled the pool (cpu_count + 4 threads; 5 WebSocket connects on a 2-vCPU box), the next tool could never start, the lock was never released, and every default-executor user server-wide hung, including event reads for unrelated conversations. /alive kept answering. - SDK: the sequential aexecute_batch path runs tools on its own one-thread pool, shut down without waiting so an interrupt never blocks the loop. - agent-server: calls that may block on a conversation's state lock run on a per-conversation executor, so waiters on one conversation no longer starve lock-free event reads on others. --- .../openhands/agent_server/event_service.py | 93 +++++++++---------- .../openhands/sdk/agent/parallel_executor.py | 31 +++++-- tests/sdk/agent/test_parallel_executor.py | 23 +++++ 3 files changed, 86 insertions(+), 61 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index 5c2718d87dc..2a3dafe8c3b 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1,14 +1,14 @@ import asyncio import time import traceback -from collections.abc import Mapping +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext, suppress from dataclasses import dataclass, field from datetime import datetime from functools import partial from pathlib import Path -from typing import cast +from typing import Any, cast from uuid import UUID, uuid4 from pydantic import ValidationError @@ -271,6 +271,11 @@ class EventService: _lease_task: asyncio.Task | None = field(default=None, init=False) _external_lease_renewal: bool = field(default=False, init=False) _run_executor: ThreadPoolExecutor | None = field(default=None, init=False) + # Runs calls that may block on this conversation's state lock, which a run + # holds across whole tool steps. On the shared default executor those + # waiters starve lock-free work server-wide (other conversations' event + # reads); here they only queue behind their own conversation. + _state_executor: ThreadPoolExecutor | None = field(default=None, init=False) _run_semaphore: asyncio.Semaphore | None = field(default=None, init=False) # Permit for this conversation's current admission session. Held for the # whole chain of runs (initial run, any re-arm for stranded input, ACP @@ -297,6 +302,11 @@ class EventService: def conversation_dir(self): return self.conversations_dir / self.stored.id.hex + async def _run_state_op[R](self, fn: Callable[..., R], *args: Any) -> R: + """Run ``fn`` off-loop on this conversation's state-lock executor.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor(self._state_executor, fn, *args) + async def load_meta(self): meta_file = self.conversation_dir / "meta.json" self.stored = StoredConversation.model_validate_json( @@ -788,8 +798,7 @@ def _get_execution_status_sync(self) -> ConversationExecutionStatus: return state.execution_status async def _get_execution_status(self) -> ConversationExecutionStatus: - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._get_execution_status_sync) + return await self._run_state_op(self._get_execution_status_sync) def _mark_error_status_sync(self) -> None: """Force the conversation into ERROR status (idempotent backstop). @@ -836,8 +845,7 @@ def _create_state_update_event_sync(self) -> ConversationStateUpdateEvent: return ConversationStateUpdateEvent.from_conversation_state(state) async def _create_state_update_event(self) -> ConversationStateUpdateEvent: - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._create_state_update_event_sync) + return await self._run_state_op(self._create_state_update_event_sync) def _event_matches_body(self, event: Event, body: str) -> bool: """Check if event's message content matches body filter (case-insensitive).""" @@ -903,8 +911,7 @@ async def send_message( if not _from_goal_loop: await self.stop_goal_loop() explicit_interrupt_generation = self._explicit_interrupt_generation - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.send_message, message) + await self._run_state_op(self._conversation.send_message, message) if run: if self._explicit_interrupt_generation != explicit_interrupt_generation: return @@ -1008,10 +1015,7 @@ def _mark_running_acp_prompt_superseded_sync(self) -> tuple[bool, bool]: return (True, False) async def _mark_running_acp_prompt_superseded(self) -> tuple[bool, bool]: - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, self._mark_running_acp_prompt_superseded_sync - ) + return await self._run_state_op(self._mark_running_acp_prompt_superseded_sync) async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: subscriber_id = self._pub_sub.subscribe(subscriber) @@ -1089,7 +1093,7 @@ def locked_on_event(): # needs a done-callback: otherwise it is collected with an unread # exception, so a dropped stats or LLM-log event leaves no signal # beyond a GC-time warning. - future = main_loop.run_in_executor(None, locked_on_event) + future = main_loop.run_in_executor(self._state_executor, locked_on_event) # Attach from the loop thread. This helper is called from foreign # threads, and `add_done_callback` on an already-done future uses # plain `call_soon`, which neither is thread-safe nor wakes an idle @@ -1240,6 +1244,11 @@ def _ensure_workspace_is_git_repo(working_dir: Path) -> None: async def start(self): # Store the main event loop for cross-thread communication self._main_loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + # Created here, not lazily: _emit_event_from_thread submits to it from + # foreign threads. + self._state_executor = ThreadPoolExecutor( + max_workers=4, thread_name_prefix="conversation-state" + ) # self.stored contains an Agent configuration we can instantiate self.conversation_dir.mkdir(parents=True, exist_ok=True) @@ -1675,10 +1684,8 @@ async def _run_and_publish(): # ConversationRunError means run()/arun() already emitted its # own event, so skip it there to avoid duplicating the error. if not isinstance(exc, ConversationRunError): - await loop.run_in_executor( - None, self._publish_error_event_sync, exc - ) - await loop.run_in_executor(None, self._mark_error_status_sync) + await self._run_state_op(self._publish_error_event_sync, exc) + await self._run_state_op(self._mark_error_status_sync) finally: settle() # Wait for all pending events to be published via @@ -2101,18 +2108,14 @@ async def reject_pending_actions(self, reason: str): """Reject all pending actions and publish updated state.""" if not self._conversation: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.reject_pending_actions, reason - ) + await self._run_state_op(self._conversation.reject_pending_actions, reason) async def pause(self): if self._conversation: self._explicit_interrupt_generation += 1 self._rerun_requested = False self._acp_internal_rerun_requested = False - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.pause) + await self._run_state_op(self._conversation.pause) # Publish state update after pause to ensure stats are updated await self._publish_state_update() @@ -2158,17 +2161,13 @@ async def update_secrets(self, secrets: dict[str, SecretValue]): if CODEX_AUTH_SECRET_NAME in self.credential_bindings: secrets = dict(secrets) secrets.pop(CODEX_AUTH_SECRET_NAME, None) - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.update_secrets, secrets) + await self._run_state_op(self._conversation.update_secrets, secrets) async def set_confirmation_policy(self, policy: ConfirmationPolicyBase): """Set the confirmation policy for the conversation.""" if not self._conversation: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.set_confirmation_policy, policy - ) + await self._run_state_op(self._conversation.set_confirmation_policy, policy) async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None: """Set the conversation's cost ceiling, taking effect from its next step. @@ -2180,9 +2179,8 @@ async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None conversation = self._conversation if conversation is None: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, setattr, conversation, "max_budget_per_run", max_budget_per_run + await self._run_state_op( + setattr, conversation, "max_budget_per_run", max_budget_per_run ) async def set_security_analyzer( @@ -2191,17 +2189,15 @@ async def set_security_analyzer( """Set the security analyzer for the conversation.""" if not self._conversation: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.set_security_analyzer, security_analyzer + await self._run_state_op( + self._conversation.set_security_analyzer, security_analyzer ) async def load_plugin(self, plugin_ref: str) -> None: """Load a marketplace plugin into the active conversation.""" if self._conversation is None: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.load_plugin, plugin_ref) + await self._run_state_op(self._conversation.load_plugin, plugin_ref) async def switch_acp_model(self, model: str) -> None: """Switch the model on an ACP conversation. @@ -2222,8 +2218,7 @@ async def switch_acp_model(self, model: str) -> None: # longer raises for a created-but-not-yet-run conversation, so a # pre-first-run switch is a normal 200 deferral, not an error. raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.switch_acp_model, model) + await self._run_state_op(self._conversation.switch_acp_model, model) async def close(self): if self.bash_event_service is not None: @@ -2257,9 +2252,8 @@ async def close(self): run_task = self._run_task if run_task is not None and not run_task.done(): if self._conversation is not None: - loop = asyncio.get_running_loop() try: - await loop.run_in_executor(None, self._conversation.pause) + await self._run_state_op(self._conversation.pause) except Exception: logger.warning( "Failed to pause conversation during close", exc_info=True @@ -2284,9 +2278,11 @@ async def close(self): await self._pub_sub.close() await self._stream_pub_sub.close() if self._conversation: - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.close) + await self._run_state_op(self._conversation.close) self._conversation = None + if self._state_executor is not None: + self._state_executor.shutdown(wait=False) + self._state_executor = None self.credential_bindings = {} if self._lease is not None and self._lease_generation is not None: @@ -2328,8 +2324,7 @@ async def ask_agent(self, question: str) -> str: if not self._conversation: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._conversation.ask_agent, question) + return await self._run_state_op(self._conversation.ask_agent, question) async def condense(self) -> None: """Force condensation of the conversation history. @@ -2353,8 +2348,7 @@ async def condense(self) -> None: f"{type(condenser).__name__} does not handle condensation requests" ) - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._conversation.condense) + return await self._run_state_op(self._conversation.condense) async def navigate_to(self, event_id: str | None) -> None: """Move the conversation HEAD to an existing event (in-place re-root). @@ -2367,10 +2361,7 @@ async def navigate_to(self, event_id: str | None) -> None: if not self._conversation: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, self._conversation.navigate_to, event_id - ) + return await self._run_state_op(self._conversation.navigate_to, event_id) def _get_agent_final_response_sync(self) -> str: """Extract the agent's final response from the active branch. diff --git a/openhands-sdk/openhands/sdk/agent/parallel_executor.py b/openhands-sdk/openhands/sdk/agent/parallel_executor.py index c32e4714156..cb510a1d76c 100644 --- a/openhands-sdk/openhands/sdk/agent/parallel_executor.py +++ b/openhands-sdk/openhands/sdk/agent/parallel_executor.py @@ -159,16 +159,27 @@ def _resolve(ae: ActionEvent) -> ToolDefinition | None: return tools.get(ae.tool_name) if tools else None if len(action_events) == 1 or self._max_workers == 1: - return [ - await self._arun_safe( - action, - tool_runner, - _resolve(action), - cancel_token, - span_owner=span_owner, - ) - for action in action_events - ] + # Not the shared default executor: arun() holds the conversation + # state lock across this await, and callers waiting on that lock + # (state snapshots, send_message) park default-executor threads. + # Once they fill it, the tool never starts and the lock is never + # released. Shut down without waiting so an interrupt that cancels + # this await does not block the event loop on the tool thread. + pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="aexecute_seq") + try: + return [ + await self._arun_safe( + action, + tool_runner, + _resolve(action), + cancel_token, + pool, + span_owner, + ) + for action in action_events + ] + finally: + pool.shutdown(wait=False) with ThreadPoolExecutor( max_workers=self._max_workers, diff --git a/tests/sdk/agent/test_parallel_executor.py b/tests/sdk/agent/test_parallel_executor.py index 8436a04c5c8..511ed57001d 100644 --- a/tests/sdk/agent/test_parallel_executor.py +++ b/tests/sdk/agent/test_parallel_executor.py @@ -1,7 +1,9 @@ """Tests for ParallelToolExecutor.""" +import asyncio import threading import time +from concurrent.futures import ThreadPoolExecutor from typing import Any from unittest.mock import MagicMock @@ -238,3 +240,24 @@ def outer_tool_runner(action: Any) -> list: ) assert results == [["inner-a", "inner-b"]] + + +async def test_async_sequential_batch_runs_without_the_default_executor(): + """arun() holds the state lock across this await; callers waiting on that + lock occupy default-executor threads. A tool queued behind them would + never start, so the sequential path must not depend on that pool.""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + release = threading.Event() + blocker = loop.run_in_executor(None, release.wait) + action: Any = MagicMock() + event = MagicMock() + try: + results = await asyncio.wait_for( + ParallelToolExecutor().aexecute_batch([action], lambda a: [event]), + timeout=5, + ) + finally: + release.set() + await blocker + assert results == [[event]] From f8a28f9d68e2c55e867b6f25e021bc0e722bf2e6 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Fri, 6 Nov 2026 17:59:48 +0100 Subject: [PATCH 2/9] perf(agent-server): resume conversations off the event loop EventService.start() built LocalConversation on the event loop. On resume that replays every persisted event on the active branch (read + validate) and rebuilds the view, so loading a long conversation froze the whole server for time proportional to its history. The workspace git probe (subprocess) also ran inline. Both now run in a worker thread. --- .../openhands/agent_server/event_service.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index 2a3dafe8c3b..1ca9567f7bd 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1267,7 +1267,7 @@ async def start(self): assert isinstance(workspace, LocalWorkspace) working_dir = Path(workspace.working_dir) working_dir.mkdir(parents=True, exist_ok=True) - self._ensure_workspace_is_git_repo(working_dir) + await asyncio.to_thread(self._ensure_workspace_is_git_repo, working_dir) # base_state.json is the single source of truth for the agent. On resume # (base_state exists) pass ``agent=None`` so LocalConversation keeps the # persisted agent. On a new conversation the creating caller supplied the @@ -1360,7 +1360,11 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: reasoning_content=reasoning if isinstance(reasoning, str) else None, ) - conversation = LocalConversation( + # Off-loop: resuming replays every persisted event on the active branch + # (read + validate), which scales with conversation length and would + # otherwise freeze the whole server while it runs. + conversation = await asyncio.to_thread( + LocalConversation, agent=agent, workspace=workspace, plugins=self.stored.plugins, From e48048d3ea3ed4a5a3b1dc28d663ceb1e918cdc7 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Fri, 6 Nov 2026 18:01:56 +0100 Subject: [PATCH 3/9] test(agent-server): state-lock waiters must not occupy the default executor --- tests/agent_server/test_event_service.py | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index 44feacddb3e..d4134bb2232 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -7,6 +7,7 @@ import threading import time import zipfile +from concurrent.futures import ThreadPoolExecutor from contextlib import suppress from datetime import UTC, datetime from pathlib import Path @@ -2795,6 +2796,42 @@ def mock_conversation_with_real_lock(self): conversation._state = state return conversation + @pytest.mark.asyncio + async def test_state_lock_waiters_leave_default_executor_free( + self, event_service, mock_conversation_with_real_lock + ): + """A run holds the state lock across whole tool steps. Requests waiting + on it must not park default-executor threads: lock-free work (event + reads of every conversation, tool calls) needs that pool.""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + event_service._conversation = mock_conversation_with_real_lock + event_service._state_executor = ThreadPoolExecutor(max_workers=2) + lock = mock_conversation_with_real_lock._state._lock + held, release = threading.Event(), threading.Event() + + def hold_lock(): + with lock: + held.set() + release.wait() + + holder = threading.Thread(target=hold_lock) + holder.start() + held.wait() + waiter = asyncio.create_task(event_service._get_execution_status()) + try: + await asyncio.sleep(0.05) + free = await asyncio.wait_for( + loop.run_in_executor(None, lambda: "free"), timeout=2 + ) + assert free == "free" + finally: + release.set() + holder.join() + status = await asyncio.wait_for(waiter, timeout=2) + assert status == ConversationExecutionStatus.IDLE + event_service._state_executor.shutdown() + @pytest.mark.asyncio async def test_concurrent_subscriptions_no_deadlock( self, event_service, mock_conversation_with_real_lock From d648dd48bfa231746ee8a3065b2e1d4720fd1b83 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Fri, 6 Nov 2026 18:11:30 +0100 Subject: [PATCH 4/9] perf(tools): stop spawning tmux to format disabled pane-pool debug logs f-string logger.debug calls rendered libtmux Pane.__repr__ eagerly, which runs tmux list-sessions + list-windows. Checkout and checkin did this on every terminal command even with DEBUG off: 4 of 11 tmux subprocesses per command. Lazy %s formatting cuts a trivial command from ~285 ms to ~218 ms p50. --- .../openhands/tools/terminal/terminal/tmux_pane_pool.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py index 058d7cf62c3..f3a5203635a 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py @@ -245,13 +245,13 @@ def checkout(self, timeout: float | None = None) -> PooledTmuxTerminal: with self._lock: if self._available: terminal = self._available.popleft() - logger.debug(f"Checked out existing pane: {terminal.pane}") + logger.debug("Checked out existing pane: %s", terminal.pane) return terminal # Create a new pane (still under max_panes thanks to semaphore) terminal = self._create_pane() self._all_panes.append(terminal) - logger.debug(f"Checked out new pane: {terminal.pane}") + logger.debug("Checked out new pane: %s", terminal.pane) return terminal def checkin(self, terminal: PooledTmuxTerminal) -> None: @@ -264,7 +264,7 @@ def checkin(self, terminal: PooledTmuxTerminal) -> None: self._available.append(terminal) self._semaphore.release() - logger.debug(f"Checked in pane: {terminal.pane}") + logger.debug("Checked in pane: %s", terminal.pane) def replace(self, old_terminal: PooledTmuxTerminal) -> PooledTmuxTerminal: """Replace a checked-out pane with a fresh one. From 4cd73205e002781317b653d5ed97a54fc4b5f5e1 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Fri, 6 Nov 2026 19:26:54 +0100 Subject: [PATCH 5/9] chore(pr): add agent-server lock-starvation and cold-load repro --- .pr/agent-server-perf/README.md | 16 ++ .pr/agent-server-perf/bench_cold_load.py | 112 +++++++++++++ .../bench_lock_starvation.py | 137 +++++++++++++++ .pr/agent-server-perf/harness.py | 150 +++++++++++++++++ .pr/agent-server-perf/mock_llm.py | 156 ++++++++++++++++++ .pr/agent-server-perf/results_cold_load.jsonl | 6 + .../results_starvation_fix12.jsonl | 6 + .../results_starvation_main.jsonl | 5 + 8 files changed, 588 insertions(+) create mode 100644 .pr/agent-server-perf/README.md create mode 100644 .pr/agent-server-perf/bench_cold_load.py create mode 100644 .pr/agent-server-perf/bench_lock_starvation.py create mode 100644 .pr/agent-server-perf/harness.py create mode 100644 .pr/agent-server-perf/mock_llm.py create mode 100644 .pr/agent-server-perf/results_cold_load.jsonl create mode 100644 .pr/agent-server-perf/results_starvation_fix12.jsonl create mode 100644 .pr/agent-server-perf/results_starvation_main.jsonl diff --git a/.pr/agent-server-perf/README.md b/.pr/agent-server-perf/README.md new file mode 100644 index 00000000000..f659f48dfc1 --- /dev/null +++ b/.pr/agent-server-perf/README.md @@ -0,0 +1,16 @@ +# Agent-server lock-starvation / cold-load repro + +Runs a real `python -m openhands.agent_server` plus a mock OpenAI-compatible LLM +(`mock_llm.py`), driven over HTTP/WebSocket. `SDK_REPO=` picks the tree whose +`.venv` runs the server (default: this repo). + +```bash +# deadlock: cpu_count=2 (6-thread default executor), 5 WS clients, 3 s tools, 2 tools/step +.venv/bin/python .pr/agent-server-perf/bench_lock_starvation.py 2 5 3 label 2 + +# cold load: build a 300-step fixture, then measure /alive during the load (3 reps) +.venv/bin/python .pr/agent-server-perf/bench_cold_load.py build 300 500 +.venv/bin/python .pr/agent-server-perf/bench_cold_load.py measure 300 500 3 label +``` + +`results_*.jsonl` are the runs quoted in the PR (main = 8dc712f, fix12 = this branch). diff --git a/.pr/agent-server-perf/bench_cold_load.py b/.pr/agent-server-perf/bench_cold_load.py new file mode 100644 index 00000000000..c1b07597f0c --- /dev/null +++ b/.pr/agent-server-perf/bench_cold_load.py @@ -0,0 +1,112 @@ +"""Scenario B: cold-loading a long conversation stalls the whole event loop. + +build: bench_cold_load.py build -> fixtures/conv--/ +measure: bench_cold_load.py measure [label] + +measure starts a fresh server on a copy of the fixture, probes /alive every +5 ms, then issues one events/search request that forces EventService.start() +(LocalConversation construction -> full event replay). Reports load latency +and the worst /alive latency seen during the load (= loop stall). +""" + +import asyncio +import json +import shutil +import statistics +import sys +import tempfile +import time +from pathlib import Path + +import httpx + +from harness import HERE, Stack, log + +FIX = HERE / "fixtures" + + +async def build(steps: int, nbytes: int): + async with Stack() as st: + cid = await st.create(f"LOOP {steps} BYTES {nbytes}") + t0 = time.monotonic() + s = await st.wait_status(cid, {"finished", "error"}, 3600) + log("built", cid, s, round(time.monotonic() - t0, 1), "s") + r = await st.client.get(f"/api/conversations/{cid}/events/count") + n_events = r.json() + dst = FIX / f"conv-{steps}-{nbytes}" + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(st.tmp / "work", dst, symlinks=True) + (dst / "meta.bench.json").write_text(json.dumps({"id": cid, "events": n_events})) + print(json.dumps({"built": steps, "events": n_events, "id": cid})) + + +async def measure_once(steps: int, nbytes: int) -> dict: + src = FIX / f"conv-{steps}-{nbytes}" + meta = json.loads((src / "meta.bench.json").read_text()) + tmp = Path(tempfile.mkdtemp(prefix="ohbench-cold-")) + shutil.copytree(src, tmp / "work", symlinks=True) + async with Stack(tmp=tmp) as st: + await asyncio.sleep(1.0) + samples: list[float] = [] + stop = asyncio.Event() + + async def probe(): + async with httpx.AsyncClient(base_url=st.base) as c: + while not stop.is_set(): + t0 = time.perf_counter() + await c.get("/alive", timeout=60) + samples.append(time.perf_counter() - t0) + await asyncio.sleep(0.005) + + pt = asyncio.create_task(probe()) + await asyncio.sleep(0.3) + base_n = len(samples) + t0 = time.perf_counter() + r = await st.client.get( + f"/api/conversations/{meta['id']}/events/search?limit=1", timeout=300 + ) + load = time.perf_counter() - t0 + r.raise_for_status() + await asyncio.sleep(0.2) + stop.set() + await pt + during = samples[base_n:] + # warm request after load, for contrast + t0 = time.perf_counter() + await st.client.get(f"/api/conversations/{meta['id']}/events/search?limit=1") + warm = time.perf_counter() - t0 + shutil.rmtree(tmp, ignore_errors=True) + return { + "events": meta["events"], + "cold_load_s": round(load, 3), + "warm_search_ms": round(warm * 1000, 1), + "alive_max_ms_during_load": round(max(during) * 1000, 1), + "alive_p50_ms_during_load": round(statistics.median(during) * 1000, 1), + } + + +async def main(): + mode = sys.argv[1] + steps = int(sys.argv[2]) + if mode == "build": + await build(steps, int(sys.argv[3])) + return + nbytes = int(sys.argv[3]) + reps = int(sys.argv[4]) + label = sys.argv[5] if len(sys.argv) > 5 else "run" + runs = [await measure_once(steps, nbytes) for _ in range(reps)] + out = { + "label": label, + "steps": steps, + "events": runs[0]["events"], + "cold_load_s_median": statistics.median(r["cold_load_s"] for r in runs), + "alive_max_ms_median": statistics.median(r["alive_max_ms_during_load"] for r in runs), + "warm_search_ms_median": statistics.median(r["warm_search_ms"] for r in runs), + "runs": runs, + } + print(json.dumps(out)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/.pr/agent-server-perf/bench_lock_starvation.py b/.pr/agent-server-perf/bench_lock_starvation.py new file mode 100644 index 00000000000..e3aa0489ee3 --- /dev/null +++ b/.pr/agent-server-perf/bench_lock_starvation.py @@ -0,0 +1,137 @@ +"""Scenario A: FIFOLock waiters vs. the shared default executor. + +Conversation A runs a step with 2 sequential `terminal` tool calls (`sleep S`). +While the first tool runs (A's arun holds the state FIFOLock on the loop +thread), K WebSocket clients connect to A. Each connect snapshots state in the +default executor and blocks on the FIFOLock. Meanwhile a probe hits a *different* +idle conversation B's event search (lock-free, but also uses the default +executor) and /alive (pure loop). + +Reports: did A finish, how long, probe latencies on B and /alive. + +usage: bench_lock_starvation.py [sleep_s] [label] +""" + +import asyncio +import json +import statistics +import sys +import time + +import httpx +import websockets + +from harness import Stack, log + + +async def probe(stack: Stack, path: str, stop: asyncio.Event, out: list, tag: str): + async with httpx.AsyncClient(base_url=stack.base) as c: + while not stop.is_set(): + t0 = time.perf_counter() + try: + r = await c.get(path, timeout=5) + ok = r.status_code == 200 + except Exception: + ok = False + dt = time.perf_counter() - t0 + out.append((time.monotonic(), dt, ok)) + await asyncio.sleep(0.05) + + +async def ws_client(stack: Stack, cid: str, opened: list, stop: asyncio.Event): + t0 = time.perf_counter() + try: + async with websockets.connect( + f"{stack.ws_base}/sockets/events/{cid}", open_timeout=60 + ) as ws: + # First frame = initial state snapshot (needs the FIFOLock). + await asyncio.wait_for(ws.recv(), timeout=60) + opened.append(time.perf_counter() - t0) + while not stop.is_set(): + with contextlib_suppress(): + await asyncio.wait_for(ws.recv(), timeout=0.5) + except Exception as e: # noqa: BLE001 + opened.append(("err", type(e).__name__)) + + +class contextlib_suppress: + def __enter__(self): + return self + + def __exit__(self, *exc): + return exc[0] is not None and issubclass(exc[0], (asyncio.TimeoutError, TimeoutError)) + + +def summarize(samples, t_from, t_to): + w = [dt for (t, dt, ok) in samples if t_from <= t <= t_to] + fails = sum(1 for (t, dt, ok) in samples if t_from <= t <= t_to and not ok) + if not w: + return {"n": 0} + return { + "n": len(w), + "p50_ms": round(statistics.median(w) * 1000, 1), + "max_ms": round(max(w) * 1000, 1), + "fails": fails, + } + + +async def main(): + cpu = int(sys.argv[1]) + K = int(sys.argv[2]) + sleep_s = float(sys.argv[3]) if len(sys.argv) > 3 else 4.0 + label = sys.argv[4] if len(sys.argv) > 4 else "run" + ntools = int(sys.argv[5]) if len(sys.argv) > 5 else 2 + async with Stack(cpu_count=cpu) as st: + b = await st.create("SLEEP 0") + assert await st.wait_status(b, {"finished"}, 60), "B did not finish" + a = await st.create(f"SLEEP {sleep_s} x{ntools}") + assert await st.wait_status(a, {"running"}, 30), "A did not start" + await asyncio.sleep(1.0) # first tool is now running; A holds the lock + + stop = asyncio.Event() + b_samples: list = [] + alive_samples: list = [] + probes = [ + asyncio.create_task(probe(st, f"/api/conversations/{b}/events/search?limit=5", stop, b_samples, "B")), + asyncio.create_task(probe(st, "/alive", stop, alive_samples, "alive")), + ] + t_start = time.monotonic() + opened: list = [] + clients = [asyncio.create_task(ws_client(st, a, opened, stop)) for _ in range(K)] + + timeout = sleep_s * 2 + 25 + done = await st.wait_status(a, {"finished", "error"}, timeout) + t_end = time.monotonic() + a_secs = round(t_end - t_start, 2) + recheck = None + if not done: + await asyncio.sleep(60) + recheck = await st.status(a) + # let ws clients get their snapshots if unblocked + await asyncio.sleep(1.0) + stop.set() + for t in clients + probes: + t.cancel() + await asyncio.gather(*clients, *probes, return_exceptions=True) + + ok_open = [x for x in opened if isinstance(x, float)] + result = { + "label": label, + "cpu_count": cpu, + "default_executor_workers": min(32, cpu + 4), + "K_ws_clients": K, + "tool_sleep_s": sleep_s, + "A_final_status": done or "STUCK (timeout)", + "A_secs_from_storm": a_secs if done else f">{timeout}", + "A_status_60s_later": recheck, + "tools_in_step": ntools, + "ws_snapshots_received": len(ok_open), + "ws_snapshot_p50_s": round(statistics.median(ok_open), 2) if ok_open else None, + "probe_B_event_search": summarize(b_samples, t_start, t_end), + "probe_alive": summarize(alive_samples, t_start, t_end), + } + print(json.dumps(result)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/.pr/agent-server-perf/harness.py b/.pr/agent-server-perf/harness.py new file mode 100644 index 00000000000..dbc61fa1804 --- /dev/null +++ b/.pr/agent-server-perf/harness.py @@ -0,0 +1,150 @@ +"""Spawn a real agent-server + mock LLM as subprocesses and drive them over HTTP/WS.""" + +import asyncio +import contextlib +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import httpx + +HERE = Path(__file__).parent +# Checkout whose .venv runs the server; defaults to the repo containing .pr/. +REPO = Path(os.environ.get("SDK_REPO", Path(__file__).resolve().parents[2])) +PY = str(REPO / ".venv/bin/python") + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +async def wait_http(url: str, timeout: float = 90) -> None: + deadline = time.monotonic() + timeout + async with httpx.AsyncClient() as c: + while time.monotonic() < deadline: + with contextlib.suppress(Exception): + r = await c.get(url, timeout=2) + if r.status_code < 500: + return + await asyncio.sleep(0.25) + raise TimeoutError(url) + + +class Stack: + def __init__(self, cpu_count: int | None = None, extra_env: dict | None = None, + stream: bool = False, tmp: Path | None = None): + self.cpu_count = cpu_count + self.extra_env = extra_env or {} + self.stream = stream + self.tmp = tmp or Path(tempfile.mkdtemp(prefix="ohbench-")) + self.llm_port = free_port() + self.port = free_port() + self.procs: list[subprocess.Popen] = [] + + @property + def base(self) -> str: + return f"http://127.0.0.1:{self.port}" + + @property + def ws_base(self) -> str: + return f"ws://127.0.0.1:{self.port}" + + async def __aenter__(self): + env = {**os.environ, "OPENHANDS_SUPPRESS_BANNER": "1", **self.extra_env} + env.pop("OH_SESSION_API_KEYS_0", None) + env.pop("SESSION_API_KEY", None) + llm_log = open(self.tmp / "llm.log", "w") + self.procs.append(subprocess.Popen( + [PY, str(HERE / "mock_llm.py"), str(self.llm_port)], + stdout=llm_log, stderr=subprocess.STDOUT, env=env, + )) + srv_env = dict(env) + if self.cpu_count: + srv_env["PYTHON_CPU_COUNT"] = str(self.cpu_count) + srv_env.setdefault("LOG_LEVEL", "WARNING") + srv_log = open(self.tmp / "server.log", "w") + (self.tmp / "work").mkdir(exist_ok=True) + self.procs.append(subprocess.Popen( + [PY, "-m", "openhands.agent_server", "--host", "127.0.0.1", + "--port", str(self.port)], + cwd=self.tmp / "work", stdout=srv_log, stderr=subprocess.STDOUT, + env=srv_env, + )) + await wait_http(f"http://127.0.0.1:{self.llm_port}/stats") + await wait_http(f"{self.base}/ready", timeout=180) + self.client = httpx.AsyncClient(base_url=self.base, timeout=120) + return self + + async def __aexit__(self, *exc): + with contextlib.suppress(Exception): + await self.client.aclose() + for p in self.procs: + p.terminate() + for p in self.procs: + with contextlib.suppress(Exception): + p.wait(timeout=10) + if p.poll() is None: + p.kill() + + def agent_spec(self) -> dict: + return { + "llm": { + "model": "openai/mock", + "base_url": f"http://127.0.0.1:{self.llm_port}/v1", + "api_key": "sk-mock", + "usage_id": "agent", + "stream": self.stream, + "num_retries": 0, + }, + "tools": [{"name": "terminal"}], + } + + async def create(self, text: str | None = None, run: bool = True) -> str: + body = { + "agent": self.agent_spec(), + "workspace": {"working_dir": str(self.tmp / "ws")}, + "autotitle": False, + } + if text is not None: + body["initial_message"] = { + "role": "user", "content": [{"type": "text", "text": text}], + "run": run, + } + r = await self.client.post("/api/conversations", json=body) + r.raise_for_status() + return r.json()["id"] + + async def status(self, cid: str) -> str: + try: + r = await self.client.get(f"/api/conversations/{cid}", timeout=5) + except httpx.TimeoutException: + return "unresponsive" + r.raise_for_status() + return r.json()["execution_status"] + + async def wait_status(self, cid: str, want: set[str], timeout: float) -> str | None: + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = await self.status(cid) + if last in want: + return last + await asyncio.sleep(0.1) + return None + + async def send(self, cid: str, text: str, run: bool = True) -> None: + r = await self.client.post( + f"/api/conversations/{cid}/events", + json={"role": "user", "content": [{"type": "text", "text": text}], "run": run}, + ) + r.raise_for_status() + + +def log(*a): + print(*a, flush=True, file=sys.stderr) diff --git a/.pr/agent-server-perf/mock_llm.py b/.pr/agent-server-perf/mock_llm.py new file mode 100644 index 00000000000..8423f407d0d --- /dev/null +++ b/.pr/agent-server-perf/mock_llm.py @@ -0,0 +1,156 @@ +"""Minimal OpenAI-compatible chat-completions server for agent-server benchmarks. + +The last *user* message is a tiny script: + + SLEEP [x] [LAT ] [TOKENS ] + +- First LLM call after the user message: return N parallel `terminal` tool calls + running `sleep ` (N defaults to 1). +- Any later call (tool results present): return a `finish` tool call. +- LAT adds artificial LLM latency per call (default 0). +- TOKENS makes the final answer a plain-text message of n words (no tools) so + streaming/large-content paths can be exercised. +""" + +import asyncio +import json +import re +import sys +import time +import uuid + +import uvicorn +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, StreamingResponse +from starlette.routing import Route + +CALLS = {"n": 0} + + +def _text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join(p.get("text", "") for p in content if isinstance(p, dict)) + return "" + + +def _plan(messages: list[dict]) -> tuple[str, dict]: + last_user_idx = max( + (i for i, m in enumerate(messages) if m.get("role") == "user"), default=-1 + ) + script = _text(messages[last_user_idx]["content"]) if last_user_idx >= 0 else "" + after = messages[last_user_idx + 1 :] + tool_results = sum(1 for m in after if m.get("role") == "tool") + m = re.search(r"SLEEP\s+([\d.]+)(?:\s+x(\d+))?", script) + lat = re.search(r"LAT\s+(\d+)", script) + toks = re.search(r"TOKENS\s+(\d+)", script) + rate = re.search(r"RATE\s+(\d+)", script) + opts = { + "lat": int(lat.group(1)) / 1000 if lat else 0.0, + "tokens": int(toks.group(1)) if toks else 0, + "rate": int(rate.group(1)) if rate else 0, + } + if m and tool_results == 0: + return "tools", {**opts, "sleep": m.group(1), "n": int(m.group(2) or 1)} + loop = re.search(r"LOOP\s+(\d+)(?:\s+BYTES\s+(\d+))?", script) + if loop and tool_results < int(loop.group(1)): + return "loop", {**opts, "bytes": int(loop.group(2) or 200), "i": tool_results} + return "finish", opts + + +def _tool_call(name: str, args: dict) -> dict: + return { + "id": f"call_{uuid.uuid4().hex[:12]}", + "type": "function", + "function": {"name": name, "arguments": json.dumps(args)}, + } + + +async def completions(request: Request): + body = await request.json() + CALLS["n"] += 1 + kind, opts = _plan(body.get("messages", [])) + if opts["lat"]: + await asyncio.sleep(opts["lat"]) + if kind == "tools": + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + _tool_call( + "terminal", + {"command": f"sleep {opts['sleep']}", "security_risk": "LOW"}, + ) + for _ in range(opts["n"]) + ], + } + elif kind == "loop": + message = { + "role": "assistant", + "content": None, + "tool_calls": [_tool_call("terminal", { + "command": f"echo step-{opts['i']}; head -c {opts['bytes']} /dev/zero | tr '\\0' x; echo", + "security_risk": "LOW", + })], + } + elif opts["tokens"]: + message = { + "role": "assistant", + "content": " ".join(f"w{i}" for i in range(opts["tokens"])), + } + else: + message = { + "role": "assistant", + "content": None, + "tool_calls": [_tool_call("finish", {"message": "done"})], + } + resp = { + "id": f"chatcmpl-{uuid.uuid4().hex[:8]}", + "object": "chat.completion", + "created": int(time.time()), + "model": body.get("model", "mock"), + "choices": [{"index": 0, "message": message, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}, + } + if not body.get("stream"): + return JSONResponse(resp) + + async def gen(): + # Stream content word-by-word; tool calls in one chunk. + base = {k: resp[k] for k in ("id", "created", "model")} + base["object"] = "chat.completion.chunk" + if message.get("content"): + for w in message["content"].split(" "): + chunk = {**base, "choices": [{"index": 0, "delta": {"content": w + " "}}]} + yield f"data: {json.dumps(chunk)}\n\n" + if opts["rate"]: + await asyncio.sleep(1 / opts["rate"]) + if message.get("tool_calls"): + tcs = [ + {**tc, "index": i} for i, tc in enumerate(message["tool_calls"]) + ] + chunk = {**base, "choices": [{"index": 0, "delta": {"role": "assistant", "tool_calls": tcs}}]} + yield f"data: {json.dumps(chunk)}\n\n" + end = {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": resp["usage"]} + yield f"data: {json.dumps(end)}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse(gen(), media_type="text/event-stream") + + +async def stats(_: Request): + return JSONResponse(CALLS) + + +app = Starlette( + routes=[ + Route("/v1/chat/completions", completions, methods=["POST"]), + Route("/chat/completions", completions, methods=["POST"]), + Route("/stats", stats), + ] +) + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=int(sys.argv[1]), log_level="warning") diff --git a/.pr/agent-server-perf/results_cold_load.jsonl b/.pr/agent-server-perf/results_cold_load.jsonl new file mode 100644 index 00000000000..05d34f9e4cd --- /dev/null +++ b/.pr/agent-server-perf/results_cold_load.jsonl @@ -0,0 +1,6 @@ +{"label": "main", "steps": 300, "events": 908, "cold_load_s_median": 0.226, "alive_max_ms_median": 106.4, "warm_search_ms_median": 3.0, "runs": [{"events": 908, "cold_load_s": 0.25, "warm_search_ms": 3.0, "alive_max_ms_during_load": 123.6, "alive_p50_ms_during_load": 1.8}, {"events": 908, "cold_load_s": 0.216, "warm_search_ms": 3.5, "alive_max_ms_during_load": 103.1, "alive_p50_ms_during_load": 1.7}, {"events": 908, "cold_load_s": 0.226, "warm_search_ms": 2.9, "alive_max_ms_during_load": 106.4, "alive_p50_ms_during_load": 1.7}]} +{"label": "fixed", "steps": 300, "events": 908, "cold_load_s_median": 0.245, "alive_max_ms_median": 8.9, "warm_search_ms_median": 4.0, "runs": [{"events": 908, "cold_load_s": 0.239, "warm_search_ms": 4.5, "alive_max_ms_during_load": 8.7, "alive_p50_ms_during_load": 1.9}, {"events": 908, "cold_load_s": 0.33, "warm_search_ms": 3.1, "alive_max_ms_during_load": 11.4, "alive_p50_ms_during_load": 1.8}, {"events": 908, "cold_load_s": 0.245, "warm_search_ms": 4.0, "alive_max_ms_during_load": 8.9, "alive_p50_ms_during_load": 1.8}]} +{"label": "main", "steps": 1500, "events": 1506, "cold_load_s_median": 0.355, "alive_max_ms_median": 198.9, "warm_search_ms_median": 3.9, "runs": [{"events": 1506, "cold_load_s": 0.617, "warm_search_ms": 4.7, "alive_max_ms_during_load": 306.3, "alive_p50_ms_during_load": 2.9}, {"events": 1506, "cold_load_s": 0.355, "warm_search_ms": 3.3, "alive_max_ms_during_load": 198.9, "alive_p50_ms_during_load": 1.7}, {"events": 1506, "cold_load_s": 0.292, "warm_search_ms": 3.9, "alive_max_ms_during_load": 163.4, "alive_p50_ms_during_load": 1.4}]} +{"label": "fixed", "steps": 1500, "events": 1506, "cold_load_s_median": 0.357, "alive_max_ms_median": 8.6, "warm_search_ms_median": 3.4, "runs": [{"events": 1506, "cold_load_s": 0.368, "warm_search_ms": 3.6, "alive_max_ms_during_load": 8.9, "alive_p50_ms_during_load": 1.8}, {"events": 1506, "cold_load_s": 0.338, "warm_search_ms": 3.2, "alive_max_ms_during_load": 6.9, "alive_p50_ms_during_load": 1.5}, {"events": 1506, "cold_load_s": 0.357, "warm_search_ms": 3.4, "alive_max_ms_during_load": 8.6, "alive_p50_ms_during_load": 1.7}]} +{"label": "main", "steps": 400, "events": 1208, "cold_load_s_median": 0.286, "alive_max_ms_median": 152.4, "warm_search_ms_median": 3.3, "runs": [{"events": 1208, "cold_load_s": 0.368, "warm_search_ms": 5.8, "alive_max_ms_during_load": 174.8, "alive_p50_ms_during_load": 2.4}, {"events": 1208, "cold_load_s": 0.268, "warm_search_ms": 3.0, "alive_max_ms_during_load": 139.8, "alive_p50_ms_during_load": 1.6}, {"events": 1208, "cold_load_s": 0.286, "warm_search_ms": 3.3, "alive_max_ms_during_load": 152.4, "alive_p50_ms_during_load": 1.5}]} +{"label": "fixed", "steps": 400, "events": 1208, "cold_load_s_median": 0.31, "alive_max_ms_median": 9.0, "warm_search_ms_median": 3.7, "runs": [{"events": 1208, "cold_load_s": 0.324, "warm_search_ms": 3.8, "alive_max_ms_during_load": 11.9, "alive_p50_ms_during_load": 1.8}, {"events": 1208, "cold_load_s": 0.297, "warm_search_ms": 3.7, "alive_max_ms_during_load": 8.1, "alive_p50_ms_during_load": 1.6}, {"events": 1208, "cold_load_s": 0.31, "warm_search_ms": 3.6, "alive_max_ms_during_load": 9.0, "alive_p50_ms_during_load": 1.7}]} diff --git a/.pr/agent-server-perf/results_starvation_fix12.jsonl b/.pr/agent-server-perf/results_starvation_fix12.jsonl new file mode 100644 index 00000000000..fbafec8b979 --- /dev/null +++ b/.pr/agent-server-perf/results_starvation_fix12.jsonl @@ -0,0 +1,6 @@ +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 4, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.83, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 4, "ws_snapshot_p50_s": 6.7, "probe_B_event_search": {"n": 118, "p50_ms": 4.8, "max_ms": 54.5, "fails": 0}, "probe_alive": {"n": 125, "p50_ms": 2.5, "max_ms": 34.8, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 5, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.65, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 5, "ws_snapshot_p50_s": 6.47, "probe_B_event_search": {"n": 118, "p50_ms": 4.1, "max_ms": 46.7, "fails": 0}, "probe_alive": {"n": 124, "p50_ms": 2.3, "max_ms": 27.2, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 7.33, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 7.16, "probe_B_event_search": {"n": 133, "p50_ms": 3.7, "max_ms": 45.0, "fails": 0}, "probe_alive": {"n": 137, "p50_ms": 2.1, "max_ms": 28.9, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 3.03, "A_status_60s_later": null, "tools_in_step": 1, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 2.96, "probe_B_event_search": {"n": 54, "p50_ms": 3.8, "max_ms": 51.6, "fails": 0}, "probe_alive": {"n": 57, "p50_ms": 2.0, "max_ms": 31.7, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 12, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.51, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 12, "ws_snapshot_p50_s": 6.42, "probe_B_event_search": {"n": 116, "p50_ms": 4.1, "max_ms": 58.0, "fails": 0}, "probe_alive": {"n": 120, "p50_ms": 2.3, "max_ms": 43.9, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 32, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.9, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 32, "ws_snapshot_p50_s": 6.82, "probe_B_event_search": {"n": 121, "p50_ms": 4.1, "max_ms": 85.8, "fails": 0}, "probe_alive": {"n": 127, "p50_ms": 2.3, "max_ms": 63.0, "fails": 0}} diff --git a/.pr/agent-server-perf/results_starvation_main.jsonl b/.pr/agent-server-perf/results_starvation_main.jsonl new file mode 100644 index 00000000000..1347626789c --- /dev/null +++ b/.pr/agent-server-perf/results_starvation_main.jsonl @@ -0,0 +1,5 @@ +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 4, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.65, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 4, "ws_snapshot_p50_s": 6.45, "probe_B_event_search": {"n": 118, "p50_ms": 4.0, "max_ms": 62.2, "fails": 0}, "probe_alive": {"n": 122, "p50_ms": 2.2, "max_ms": 38.0, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 5, "tool_sleep_s": 3.0, "A_final_status": "STUCK (timeout)", "A_secs_from_storm": ">31.0", "A_status_60s_later": "running", "tools_in_step": 2, "ws_snapshots_received": 0, "ws_snapshot_p50_s": null, "probe_B_event_search": {"n": 6, "p50_ms": 5003.7, "max_ms": 5035.1, "fails": 6}, "probe_alive": {"n": 581, "p50_ms": 2.3, "max_ms": 49.8, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "STUCK (timeout)", "A_secs_from_storm": ">31.0", "A_status_60s_later": "unresponsive", "tools_in_step": 2, "ws_snapshots_received": 0, "ws_snapshot_p50_s": null, "probe_B_event_search": {"n": 6, "p50_ms": 5015.0, "max_ms": 5354.3, "fails": 6}, "probe_alive": {"n": 482, "p50_ms": 4.7, "max_ms": 374.7, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 3.14, "A_status_60s_later": null, "tools_in_step": 1, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 3.05, "probe_B_event_search": {"n": 1, "p50_ms": 3090.2, "max_ms": 3090.2, "fails": 0}, "probe_alive": {"n": 56, "p50_ms": 2.7, "max_ms": 53.2, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 12, "tool_sleep_s": 3.0, "A_final_status": "STUCK (timeout)", "A_secs_from_storm": ">31.0", "A_status_60s_later": "running", "tools_in_step": 2, "ws_snapshots_received": 0, "ws_snapshot_p50_s": null, "probe_B_event_search": {"n": 6, "p50_ms": 5006.2, "max_ms": 5035.2, "fails": 6}, "probe_alive": {"n": 557, "p50_ms": 2.8, "max_ms": 40.3, "fails": 0}} From 7f200dce21a12ebf46780d66a338b047470a1c59 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Sat, 7 Nov 2026 23:06:34 +0100 Subject: [PATCH 6/9] fix(tools): detect a shell-killing command without the debug-log repr The pool's shell-exit recovery only fired because checkin() rendered Pane.__repr__ eagerly in an f-string debug log; that repr ran tmux and raised once the shell had exited. With lazy logging, `exit 7` just timed out. Refresh the pane explicitly when a command does not finish (and the terminal was not closed on purpose), so completed commands still spawn no extra tmux process. --- openhands-tools/openhands/tools/terminal/impl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openhands-tools/openhands/tools/terminal/impl.py b/openhands-tools/openhands/tools/terminal/impl.py index 598ee9dfa72..a38df8414cf 100644 --- a/openhands-tools/openhands/tools/terminal/impl.py +++ b/openhands-tools/openhands/tools/terminal/impl.py @@ -516,6 +516,12 @@ def _execute_pooled( ) self._export_envs(cmd_action, conversation, session=session) observation = session.execute(cmd_action) + if observation.exit_code == -1 and not handle.terminal.closed: + # The command did not finish and we did not close the + # terminal. If it exited the shell, the pane (or the whole + # tmux server) is gone and refresh() raises a recoverable + # tmux error handled below. + handle.terminal.pane.refresh() if reset_text is not None: observation = observation.model_copy( From 5fd886a82809e67e1769f3745cb6597ec4fabecc Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Wed, 11 Nov 2026 15:42:38 +0100 Subject: [PATCH 7/9] fix(agent-server): keep WebSocket snapshot waits off the default executor subscribe_to_events() took the state lock on the shared default executor again once #6577 was merged in, so six WebSocket connects during a tool step still starved the lock-free work of every other conversation. switch_acp_effort(), added on main, had the same shape. Both now run on the per-conversation state executor, and the lock-waiter test also covers the real subscription path. --- .../openhands/agent_server/event_service.py | 8 +++----- tests/agent_server/test_event_service.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index 6c46f607988..36c320c9f42 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1043,9 +1043,8 @@ async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: # conversation's synchronous FIFOLock cannot block the server event loop. log_length: int | None = None if self._conversation: - loop = asyncio.get_running_loop() - state_update_event, log_length = await loop.run_in_executor( - None, self._create_initial_state_sync + state_update_event, log_length = await self._run_state_op( + self._create_initial_state_sync ) else: state_update_event = ConversationStateUpdateEvent( @@ -2269,8 +2268,7 @@ async def switch_acp_effort(self, effort: str) -> None: """ if self._conversation is None: raise ValueError("inactive_service") - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.switch_acp_effort, effort) + await self._run_state_op(self._conversation.switch_acp_effort, effort) async def close(self): if self.bash_event_service is not None: diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index d4134bb2232..5151d48c6c5 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -2797,8 +2797,20 @@ def mock_conversation_with_real_lock(self): return conversation @pytest.mark.asyncio + @pytest.mark.parametrize( + "wait_on_state_lock", + [ + pytest.param( + lambda service: service._get_execution_status(), id="execution-status" + ), + pytest.param( + lambda service: service.subscribe_to_events(AsyncMock()), + id="websocket-subscribe", + ), + ], + ) async def test_state_lock_waiters_leave_default_executor_free( - self, event_service, mock_conversation_with_real_lock + self, event_service, mock_conversation_with_real_lock, wait_on_state_lock ): """A run holds the state lock across whole tool steps. Requests waiting on it must not park default-executor threads: lock-free work (event @@ -2818,7 +2830,7 @@ def hold_lock(): holder = threading.Thread(target=hold_lock) holder.start() held.wait() - waiter = asyncio.create_task(event_service._get_execution_status()) + waiter = asyncio.create_task(wait_on_state_lock(event_service)) try: await asyncio.sleep(0.05) free = await asyncio.wait_for( @@ -2828,8 +2840,7 @@ def hold_lock(): finally: release.set() holder.join() - status = await asyncio.wait_for(waiter, timeout=2) - assert status == ConversationExecutionStatus.IDLE + await asyncio.wait_for(waiter, timeout=2) event_service._state_executor.shutdown() @pytest.mark.asyncio From 69decc77a737d33a32692861020d0901758f9a46 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Wed, 11 Nov 2026 16:27:02 +0100 Subject: [PATCH 8/9] fix(tools): notice a dead pooled shell at the next screen read tmux reports a vanished pane or server on capture-pane's stderr instead of raising, so a shell that died while idle looked like a silent command and the pool only recovered once the command timeout ran out (30 s by default). The pooled terminal now raises that error from read_screen(), which the executor's existing recovery handles before the command is sent. That replaces the refresh() probe that ran after the timeout. --- openhands-tools/openhands/tools/terminal/impl.py | 7 +------ .../tools/terminal/terminal/tmux_pane_pool.py | 10 ++++++++++ tests/tools/terminal/test_pool_integration.py | 15 +++++++++++++++ tests/tools/terminal/test_tmux_pane_pool.py | 7 ++++++- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/openhands-tools/openhands/tools/terminal/impl.py b/openhands-tools/openhands/tools/terminal/impl.py index ad35510f07d..b500d201c40 100644 --- a/openhands-tools/openhands/tools/terminal/impl.py +++ b/openhands-tools/openhands/tools/terminal/impl.py @@ -53,6 +53,7 @@ _TMUX_RECOVERABLE_ERROR_MARKERS = ( "no server running", "can't find session", + "can't find pane", "could not find window_id", "could not find pane_id", ) @@ -541,12 +542,6 @@ def _execute_pooled( ) self._export_envs(cmd_action, conversation, session=session) observation = session.execute(cmd_action) - if observation.exit_code == -1 and not handle.terminal.closed: - # The command did not finish and we did not close the - # terminal. If it exited the shell, the pane (or the whole - # tmux server) is gone and refresh() raises a recoverable - # tmux error handled below. - handle.terminal.pane.refresh() if session.is_running(): self._unfinished_pane = handle.terminal elif self._unfinished_pane is handle.terminal: diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py index e3ba08691df..c50855147ad 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py @@ -17,6 +17,7 @@ from typing import Final import libtmux +from libtmux.exc import LibTmuxException from openhands.sdk.logger import get_logger from openhands.tools.terminal.constants import ( @@ -59,6 +60,15 @@ def close(self) -> None: self.window.kill() self._closed = True + def read_screen(self) -> str: + # tmux reports a vanished pane or server on stderr instead of raising. + # Raise it, so the pool recovers at the next read rather than waiting + # out the command timeout on a screen that will never change. + result = self.pane.cmd("capture-pane", "-J", "-pS", "-") + if result.returncode != 0: + raise LibTmuxException(*result.stderr) + return "\n".join(line.rstrip() for line in result.stdout) + @dataclass(slots=True) class PaneHandle: diff --git a/tests/tools/terminal/test_pool_integration.py b/tests/tools/terminal/test_pool_integration.py index 07e0458a8d4..1b13602b731 100644 --- a/tests/tools/terminal/test_pool_integration.py +++ b/tests/tools/terminal/test_pool_integration.py @@ -127,6 +127,21 @@ def test_shell_exit_returns_actionable_error_and_rebuilds_pool(self, pool_execut assert after.exit_code == 0 assert "after_rebuild" in after.text + def test_shell_that_died_while_idle_is_noticed_before_the_timeout( + self, pool_executor + ): + pool_executor(TerminalAction(command="(sleep 1; kill -9 $$) &", timeout=5.0)) + time.sleep(2.5) + + started = time.monotonic() + obs = pool_executor(TerminalAction(command="echo after", timeout=30.0)) + + assert time.monotonic() - started < 10 + assert obs.is_error + assert "rebuilt the terminal pool" in obs.text + after = pool_executor(TerminalAction(command="echo after_rebuild", timeout=5.0)) + assert "after_rebuild" in after.text + def test_reset_after_shell_exit_uses_rebuilt_pool(self, pool_executor): obs = pool_executor(TerminalAction(command="exit 0", timeout=1.0)) assert obs.is_error diff --git a/tests/tools/terminal/test_tmux_pane_pool.py b/tests/tools/terminal/test_tmux_pane_pool.py index dd16e0d306c..92e2a2eee64 100644 --- a/tests/tools/terminal/test_tmux_pane_pool.py +++ b/tests/tools/terminal/test_tmux_pane_pool.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +from libtmux.exc import LibTmuxException from openhands.tools.terminal.constants import ( TMUX_SESSION_HEIGHT, @@ -306,7 +307,11 @@ def test_stale_terminal_cannot_access_restarted_server( assert time.monotonic() < deadline time.sleep(0.05) - if operation == "read": + if operation == "read" and pooled: + # A pooled pane reports its dead server so the pool can recover. + with pytest.raises(LibTmuxException, match="no server running"): + stale.read_screen() + elif operation == "read": assert "SECOND_CONVERSATION_MARKER" not in stale.read_screen() elif operation == "write": marker = tmp_path / "wrong-conversation" From 86daaddae753480578e3331f919a33c7b2f67383 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Wed, 18 Nov 2026 16:22:07 +0100 Subject: [PATCH 9/9] fix(agent-server): keep REST state-lock waits off the default executor `update_conversation` waited on the state lock on asyncio's default executor for the tag update and the webhook snapshot, and `fork_conversation` did the same for the source conversation's fork. During a tool step six tag PATCHes took every default-executor thread, so event reads of unrelated conversations stalled (live bench: 4 requests in 6.5 s, one 5 s timeout; with this change 110 requests, p50 2.7 ms, none failed). `EventService.run_state_op` is now public and `ConversationService` uses it for all three calls. The fork keeps `asyncio.to_thread`'s context copy. The new test fails when the tag update goes back to the default executor. The lock-starvation bench gains a `patch` client mode. --- .pr/agent-server-perf/README.md | 3 + .../bench_lock_starvation.py | 23 ++++++- .../results_starvation_patch.jsonl | 2 + openhands-agent-server/AGENTS.md | 2 +- .../agent_server/conversation_service.py | 26 ++++---- .../openhands/agent_server/event_service.py | 58 ++++++++--------- .../agent_server/test_conversation_service.py | 63 +++++++++++++++++++ 7 files changed, 134 insertions(+), 43 deletions(-) create mode 100644 .pr/agent-server-perf/results_starvation_patch.jsonl diff --git a/.pr/agent-server-perf/README.md b/.pr/agent-server-perf/README.md index f659f48dfc1..227f5766f31 100644 --- a/.pr/agent-server-perf/README.md +++ b/.pr/agent-server-perf/README.md @@ -14,3 +14,6 @@ Runs a real `python -m openhands.agent_server` plus a mock OpenAI-compatible LLM ``` `results_*.jsonl` are the runs quoted in the PR (main = 8dc712f, fix12 = this branch). +`results_starvation_patch.jsonl` sends 6 tag PATCHes instead of WebSocket connects +(`... bench_lock_starvation.py 2 6 3 label 2 patch`); `patch-main-cs` is this branch +with main's `conversation_service.py`. diff --git a/.pr/agent-server-perf/bench_lock_starvation.py b/.pr/agent-server-perf/bench_lock_starvation.py index e3aa0489ee3..664b3ffa7e5 100644 --- a/.pr/agent-server-perf/bench_lock_starvation.py +++ b/.pr/agent-server-perf/bench_lock_starvation.py @@ -9,7 +9,9 @@ Reports: did A finish, how long, probe latencies on B and /alive. -usage: bench_lock_starvation.py [sleep_s] [label] +usage: bench_lock_starvation.py [sleep_s] [label] [ntools] [ws|patch] + +`patch` sends K tag PATCHes to A instead of WebSocket connects. """ import asyncio @@ -54,6 +56,20 @@ async def ws_client(stack: Stack, cid: str, opened: list, stop: asyncio.Event): opened.append(("err", type(e).__name__)) +async def patch_client(stack: Stack, cid: str, opened: list, stop: asyncio.Event): + # A tag PATCH updates state and composes the webhook snapshot under the lock. + t0 = time.perf_counter() + try: + async with httpx.AsyncClient(base_url=stack.base) as c: + r = await c.patch( + f"/api/conversations/{cid}", json={"tags": {"bench": "x"}}, timeout=60 + ) + r.raise_for_status() + opened.append(time.perf_counter() - t0) + except Exception as e: # noqa: BLE001 + opened.append(("err", type(e).__name__)) + + class contextlib_suppress: def __enter__(self): return self @@ -81,6 +97,8 @@ async def main(): sleep_s = float(sys.argv[3]) if len(sys.argv) > 3 else 4.0 label = sys.argv[4] if len(sys.argv) > 4 else "run" ntools = int(sys.argv[5]) if len(sys.argv) > 5 else 2 + mode = sys.argv[6] if len(sys.argv) > 6 else "ws" + client = {"ws": ws_client, "patch": patch_client}[mode] async with Stack(cpu_count=cpu) as st: b = await st.create("SLEEP 0") assert await st.wait_status(b, {"finished"}, 60), "B did not finish" @@ -97,7 +115,7 @@ async def main(): ] t_start = time.monotonic() opened: list = [] - clients = [asyncio.create_task(ws_client(st, a, opened, stop)) for _ in range(K)] + clients = [asyncio.create_task(client(st, a, opened, stop)) for _ in range(K)] timeout = sleep_s * 2 + 25 done = await st.wait_status(a, {"finished", "error"}, timeout) @@ -119,6 +137,7 @@ async def main(): "label": label, "cpu_count": cpu, "default_executor_workers": min(32, cpu + 4), + "client_mode": mode, "K_ws_clients": K, "tool_sleep_s": sleep_s, "A_final_status": done or "STUCK (timeout)", diff --git a/.pr/agent-server-perf/results_starvation_patch.jsonl b/.pr/agent-server-perf/results_starvation_patch.jsonl new file mode 100644 index 00000000000..06f692fa353 --- /dev/null +++ b/.pr/agent-server-perf/results_starvation_patch.jsonl @@ -0,0 +1,2 @@ +{"label": "patch-fix", "cpu_count": 2, "default_executor_workers": 6, "client_mode": "patch", "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.01, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 5.91, "probe_B_event_search": {"n": 110, "p50_ms": 2.7, "max_ms": 64.7, "fails": 0}, "probe_alive": {"n": 115, "p50_ms": 1.3, "max_ms": 34.7, "fails": 0}} +{"label": "patch-main-cs", "cpu_count": 2, "default_executor_workers": 6, "client_mode": "patch", "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.55, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 6.42, "probe_B_event_search": {"n": 4, "p50_ms": 673.4, "max_ms": 5036.8, "fails": 1}, "probe_alive": {"n": 124, "p50_ms": 1.5, "max_ms": 34.1, "fails": 0}} diff --git a/openhands-agent-server/AGENTS.md b/openhands-agent-server/AGENTS.md index bba83b57e8e..c6b152d18df 100644 --- a/openhands-agent-server/AGENTS.md +++ b/openhands-agent-server/AGENTS.md @@ -123,7 +123,7 @@ feature map (`control-agent-server map coverage`). - `ConversationState` uses a synchronous `FIFOLock`. In async agent-server code, never do `with conversation._state` directly on the event loop when the conversation may be running. - WebSocket reconnects call `EventService.subscribe_to_events()` immediately; if initial state snapshot creation blocks on the state lock in async context, the whole FastAPI event loop can stop serving `/ready` and similar probes. - The same rule applies to metadata updates in `ConversationService.update_conversation()`: keep the locked mutation/snapshot semantics, but move the synchronous lock wait into a worker thread first. -- In async routes/services, move state-lock acquisition into `run_in_executor(...)` (or another worker-thread boundary) before awaiting network I/O. +- In async routes/services, move state-lock acquisition into `EventService.run_state_op(...)` before awaiting network I/O. It runs on the conversation's own executor: a run holds the lock for a whole tool step, and waiters on asyncio's default executor (`run_in_executor(None, ...)`, `asyncio.to_thread`) take the threads that event reads of every conversation need. ## REST API compatibility & deprecation policy diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 6942f67ac2c..ee6a501fdb6 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -1,4 +1,5 @@ import asyncio +import contextvars import importlib import json import logging @@ -7,6 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager, suppress from dataclasses import dataclass, field +from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any, cast from uuid import UUID, uuid4 @@ -2000,7 +2002,6 @@ async def update_conversation( if event_service is None: return False - loop = asyncio.get_running_loop() state = await event_service.get_state() update: dict[str, Any] = {"updated_at": utc_now()} if request.title is not None: @@ -2015,8 +2016,8 @@ async def update_conversation( if request.tags is not None: # Keep the persisted ConversationState update under the state lock so # autosave and state-change callbacks observe a consistent mutation. - state = await loop.run_in_executor( - None, _update_state_tags_sync, state, request.tags + state = await event_service.run_state_op( + _update_state_tags_sync, state, request.tags ) if budget_updated: # base_state.json owns the budget; the event service is loaded @@ -2030,8 +2031,8 @@ async def update_conversation( # Notify conversation webhooks about the updated conversation. Compose the # full-state snapshot under the state lock, but do the synchronous wait in a # worker thread so metadata updates cannot block the FastAPI event loop. - conversation_info = await loop.run_in_executor( - None, _compose_webhook_conversation_info_sync, event_service.stored, state + conversation_info = await event_service.run_state_op( + _compose_webhook_conversation_info_sync, event_service.stored, state ) await self._notify_conversation_webhooks(conversation_info) @@ -2146,12 +2147,15 @@ async def fork_conversation( raise ValueError(f"Conversation with id {fork_id} already exists") # fork() deep-copies events and state into ``fork_dir``. - fork_conv = await asyncio.to_thread( - source_conversation.fork, - conversation_id=fork_conv_id, - tags=tags, - reset_metrics=reset_metrics, - from_event_id=from_event_id, + fork_conv = await source_service.run_state_op( + partial( + contextvars.copy_context().run, + source_conversation.fork, + conversation_id=fork_conv_id, + tags=tags, + reset_metrics=reset_metrics, + from_event_id=from_event_id, + ) ) # Extract the persisted data, then discard the temporary conversation. fork_agent = cast(AgentBase, fork_conv.agent) diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index 8713439d200..a877d6ecc7d 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -349,7 +349,7 @@ class EventService: def conversation_dir(self): return self.conversations_dir / self.stored.id.hex - async def _run_state_op[R](self, fn: Callable[..., R], *args: Any) -> R: + async def run_state_op[R](self, fn: Callable[..., R], *args: Any) -> R: """Run ``fn``, which acquires the state lock, off the event loop.""" loop = asyncio.get_running_loop() return await loop.run_in_executor(self._state_executor, fn, *args) @@ -539,7 +539,7 @@ def _update() -> None: if isinstance(agent, ACPAgent): agent.restart_for_updated_credentials(secrets) - await self._run_state_op(contextvars.copy_context().run, _update) + await self.run_state_op(contextvars.copy_context().run, _update) await self.save_meta( self.stored.model_copy( update={"secrets": {**self.stored.secrets, **secrets}} @@ -845,7 +845,7 @@ def _get_execution_status_sync(self) -> ConversationExecutionStatus: return state.execution_status async def _get_execution_status(self) -> ConversationExecutionStatus: - return await self._run_state_op(self._get_execution_status_sync) + return await self.run_state_op(self._get_execution_status_sync) def _mark_error_status_sync(self) -> None: """Force the conversation into ERROR status (idempotent backstop). @@ -904,7 +904,7 @@ def _create_initial_state_sync( return self._create_state_update_event_sync(), len(state.events) async def _create_state_update_event(self) -> ConversationStateUpdateEvent: - return await self._run_state_op(self._create_state_update_event_sync) + return await self.run_state_op(self._create_state_update_event_sync) def _event_matches_body(self, event: Event, body: str) -> bool: """Check if event's message content matches body filter (case-insensitive).""" @@ -975,7 +975,7 @@ async def send_message( if _from_goal_loop else self._conversation.send_message ) - await self._run_state_op(send_message, message) + await self.run_state_op(send_message, message) if run: if self._explicit_interrupt_generation != explicit_interrupt_generation: return @@ -1079,7 +1079,7 @@ def _mark_running_acp_prompt_superseded_sync(self) -> tuple[bool, bool]: return (True, False) async def _mark_running_acp_prompt_superseded(self) -> tuple[bool, bool]: - return await self._run_state_op(self._mark_running_acp_prompt_superseded_sync) + return await self.run_state_op(self._mark_running_acp_prompt_superseded_sync) async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: subscriber_id = self._pub_sub.subscribe(subscriber) @@ -1089,7 +1089,7 @@ async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: # conversation's synchronous FIFOLock cannot block the server event loop. log_length: int | None = None if self._conversation: - state_update_event, log_length = await self._run_state_op( + state_update_event, log_length = await self.run_state_op( self._create_initial_state_sync ) else: @@ -1566,7 +1566,7 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: # still active was left by a shutdown or crash mid-loop (close() skips # the status on purpose). Record it as interrupted so it reads as # stopped and stays resumable. - await self._run_state_op(self._reconcile_stale_goal_status) + await self.run_state_op(self._reconcile_stale_goal_status) # Publish initial state update await self._publish_state_update() @@ -1779,8 +1779,8 @@ async def _run_and_publish(): # ConversationRunError means run()/arun() already emitted its # own event, so skip it there to avoid duplicating the error. if not isinstance(exc, ConversationRunError): - await self._run_state_op(self._publish_error_event_sync, exc) - await self._run_state_op(self._mark_error_status_sync) + await self.run_state_op(self._publish_error_event_sync, exc) + await self.run_state_op(self._mark_error_status_sync) finally: settle() # Wait for all pending events to be published via @@ -1943,7 +1943,7 @@ def _require_answered() -> None: if find_pending_ask(conversation._state.active_branch()) is not None: raise ValueError("pending_ask") - await self._run_state_op(_require_answered) + await self.run_state_op(_require_answered) if self._closing: raise ServiceInactive() # Reserve capacity before spawning the loop so a full server refuses @@ -2171,7 +2171,7 @@ async def resume_goal_loop( """ if not self._conversation or self._closing: raise ServiceInactive() - last = await self._run_state_op(self._last_goal_loop_status) + last = await self.run_state_op(self._last_goal_loop_status) if last is None or last.get("status") in ("complete", "capped"): raise ValueError("no_resumable_goal") if judge_llm is None: @@ -2236,7 +2236,7 @@ async def react( if not self._conversation: raise ServiceInactive() conversation = self._conversation - return await self._run_state_op( + return await self.run_state_op( partial(conversation.react, emoji, target_event_id, removed=removed) ) @@ -2253,7 +2253,7 @@ async def answer_ask( """ if not self._conversation: raise ServiceInactive() - answer, message = await self._run_state_op( + answer, message = await self.run_state_op( self._conversation.answer_ask, ask_id, option_index, text ) await self.send_message(message, run=True) @@ -2263,14 +2263,14 @@ async def reject_pending_actions(self, reason: str): """Reject all pending actions and publish updated state.""" if not self._conversation: raise ServiceInactive() - await self._run_state_op(self._conversation.reject_pending_actions, reason) + await self.run_state_op(self._conversation.reject_pending_actions, reason) async def pause(self): if self._conversation: self._explicit_interrupt_generation += 1 self._rerun_requested = False self._acp_internal_rerun_requested = False - await self._run_state_op(self._conversation.pause) + await self.run_state_op(self._conversation.pause) # Publish state update after pause to ensure stats are updated await self._publish_state_update() @@ -2306,7 +2306,7 @@ async def add_mcp_servers(self, servers: dict[str, MCPServer]) -> None: """Add modern MCP servers to the conversation.""" if not self._conversation: raise ServiceInactive() - await self._run_state_op(self._conversation.add_mcp_servers, servers) + await self.run_state_op(self._conversation.add_mcp_servers, servers) await self._publish_state_update() async def update_secrets(self, secrets: dict[str, SecretValue]): @@ -2323,13 +2323,13 @@ async def update_secrets(self, secrets: dict[str, SecretValue]): if CODEX_AUTH_SECRET_NAME in self.credential_bindings: secrets = dict(secrets) secrets.pop(CODEX_AUTH_SECRET_NAME, None) - await self._run_state_op(self._conversation.update_secrets, secrets) + await self.run_state_op(self._conversation.update_secrets, secrets) async def set_confirmation_policy(self, policy: ConfirmationPolicyBase): """Set the confirmation policy for the conversation.""" if not self._conversation: raise ServiceInactive() - await self._run_state_op(self._conversation.set_confirmation_policy, policy) + await self.run_state_op(self._conversation.set_confirmation_policy, policy) async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None: """Set the conversation's cost ceiling, taking effect from its next step. @@ -2341,7 +2341,7 @@ async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None conversation = self._conversation if conversation is None: raise ServiceInactive() - await self._run_state_op( + await self.run_state_op( setattr, conversation, "max_budget_per_run", max_budget_per_run ) @@ -2351,7 +2351,7 @@ async def set_security_analyzer( """Set the security analyzer for the conversation.""" if not self._conversation: raise ServiceInactive() - await self._run_state_op( + await self.run_state_op( self._conversation.set_security_analyzer, security_analyzer ) @@ -2359,7 +2359,7 @@ async def load_plugin(self, plugin_ref: str) -> None: """Load a marketplace plugin into the active conversation.""" if self._conversation is None: raise ServiceInactive() - await self._run_state_op(self._conversation.load_plugin, plugin_ref) + await self.run_state_op(self._conversation.load_plugin, plugin_ref) async def switch_acp_model(self, model: str) -> None: """Switch the model on an ACP conversation. @@ -2380,7 +2380,7 @@ async def switch_acp_model(self, model: str) -> None: # a created-but-not-yet-run conversation, so a pre-first-run switch # is a normal 200 deferral, not an error. raise ServiceInactive() - await self._run_state_op(self._conversation.switch_acp_model, model) + await self.run_state_op(self._conversation.switch_acp_model, model) async def switch_acp_effort(self, effort: str) -> None: """Switch the reasoning effort on an ACP conversation. @@ -2390,7 +2390,7 @@ async def switch_acp_effort(self, effort: str) -> None: """ if self._conversation is None: raise ServiceInactive() - await self._run_state_op(self._conversation.switch_acp_effort, effort) + await self.run_state_op(self._conversation.switch_acp_effort, effort) async def close_connections(self, code: int, reason: str) -> None: """Close the client sockets subscribed to this conversation.""" @@ -2429,7 +2429,7 @@ async def close(self): if run_task is not None and not run_task.done(): if self._conversation is not None: try: - await self._run_state_op(self._conversation.pause) + await self.run_state_op(self._conversation.pause) except Exception: logger.warning( "Failed to pause conversation during close", exc_info=True @@ -2454,7 +2454,7 @@ async def close(self): await self._pub_sub.close() await self._stream_pub_sub.close() if self._conversation: - await self._run_state_op(self._conversation.close) + await self.run_state_op(self._conversation.close) self._conversation = None if self._state_executor is not None: self._state_executor.shutdown(wait=False) @@ -2500,7 +2500,7 @@ async def ask_agent(self, question: str) -> str: if not self._conversation: raise ServiceInactive() - return await self._run_state_op(self._conversation.ask_agent, question) + return await self.run_state_op(self._conversation.ask_agent, question) async def condense(self) -> None: """Force condensation of the conversation history. @@ -2524,7 +2524,7 @@ async def condense(self) -> None: f"{type(condenser).__name__} does not handle condensation requests" ) - return await self._run_state_op(self._conversation.condense) + return await self.run_state_op(self._conversation.condense) async def navigate_to(self, event_id: str | None) -> None: """Move the conversation HEAD to an existing event (in-place re-root). @@ -2537,7 +2537,7 @@ async def navigate_to(self, event_id: str | None) -> None: if not self._conversation: raise ServiceInactive() - return await self._run_state_op(self._conversation.navigate_to, event_id) + return await self.run_state_op(self._conversation.navigate_to, event_id) def _get_agent_final_response_sync(self) -> str: """Extract the agent's final response from the active branch. diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 38daad6532a..d8f0aae66f0 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -4,7 +4,9 @@ import tempfile import threading import time +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +from functools import partial from pathlib import Path from typing import Any, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -2972,6 +2974,7 @@ async def test_update_conversation_tags_uses_state_lock( ): """Test that tag updates hold the ConversationState lock.""" mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = sample_stored_conversation _adopt_saved_meta(mock_service) mock_state = ConversationState( @@ -3007,6 +3010,7 @@ async def test_update_conversation_tags_wait_does_not_block_event_loop( ): """Waiting on the state lock must not stall unrelated async work.""" mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = sample_stored_conversation _adopt_saved_meta(mock_service) state = ConversationState( @@ -3068,6 +3072,63 @@ async def release_after_delay() -> None: "state lock" ) + @pytest.mark.asyncio + async def test_update_conversation_tags_wait_leaves_default_executor_free( + self, conversation_service, sample_stored_conversation + ): + """A run holds the state lock across whole tool steps. A tag PATCH + waiting on it must not park a default-executor thread: event reads of + every conversation need that pool.""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + state_executor = ThreadPoolExecutor(max_workers=1) + mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = partial( + loop.run_in_executor, state_executor + ) + mock_service.stored = sample_stored_conversation + _adopt_saved_meta(mock_service) + state = ConversationState( + id=sample_stored_conversation.id, + agent=_sample_agent(), + workspace=sample_stored_conversation.workspace, + execution_status=ConversationExecutionStatus.RUNNING, + confirmation_policy=sample_stored_conversation.confirmation_policy, + ) + mock_service.get_state.return_value = state + conversation_id = sample_stored_conversation.id + conversation_service._event_services[conversation_id] = mock_service + held, release = threading.Event(), threading.Event() + + def hold_state_lock() -> None: + with state: + held.set() + release.wait() + + holder = threading.Thread(target=hold_state_lock) + holder.start() + assert held.wait(timeout=1.0) + with patch.object( + conversation_service, "_notify_conversation_webhooks", new=AsyncMock() + ): + update = asyncio.create_task( + conversation_service.update_conversation( + conversation_id, UpdateConversationRequest(tags={"env": "prod"}) + ) + ) + try: + await asyncio.sleep(0.05) + free = await asyncio.wait_for( + loop.run_in_executor(None, lambda: "free"), timeout=2 + ) + assert free == "free" + finally: + release.set() + holder.join() + assert await asyncio.wait_for(update, timeout=2) + state_executor.shutdown() + assert state.tags == {"env": "prod"} + @pytest.mark.asyncio async def test_update_conversation_not_found(self, conversation_service): """Test updating a non-existent conversation returns False.""" @@ -3095,6 +3156,7 @@ async def test_update_conversation_notifies_webhooks( """Test that updating a conversation triggers webhook notifications.""" # Create mock event service mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = sample_stored_conversation _adopt_saved_meta(mock_service) mock_state = ConversationState( @@ -3143,6 +3205,7 @@ async def test_update_acp_conversation_notifies_webhooks_with_acp_shape( updated_at=datetime(2025, 1, 1, 12, 30, 0, tzinfo=UTC), ) mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = stored_conversation _adopt_saved_meta(mock_service) mock_state = ConversationState(