From e001ab1f4f3673322856b5e7627eab55224e2d10 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Tue, 17 Nov 2026 15:52:53 +0100 Subject: [PATCH] fix(agent-server): keep state-lock waiters off the default executor arun() holds a conversation's state lock for a whole tool step. Requests that wait on that lock (WebSocket subscribe snapshot, send_message, pause, PATCH, stats events) ran on asyncio's shared default executor, and so did each tool call of a sequential batch. Once the waiters filled that pool, the next tool call could not start, the lock was never released, and every other default-executor user (event reads of all conversations) hung. - SDK: the sequential aexecute_batch path runs tools on its own one-thread pool. - agent-server: calls that acquire a conversation's state lock run on a per-conversation executor (EventService.run_state_op), created in start() and shut down in close(). Part of #6553 --- openhands-agent-server/AGENTS.md | 2 +- .../agent_server/conversation_service.py | 26 ++-- .../openhands/agent_server/event_service.py | 131 ++++++++---------- .../openhands/sdk/agent/parallel_executor.py | 38 +++-- .../agent_server/test_conversation_service.py | 4 + tests/agent_server/test_event_service.py | 53 +++++++ tests/sdk/agent/test_parallel_executor.py | 24 ++++ 7 files changed, 179 insertions(+), 99 deletions(-) diff --git a/openhands-agent-server/AGENTS.md b/openhands-agent-server/AGENTS.md index bba83b57e8e..c6b152d18df 100644 --- a/openhands-agent-server/AGENTS.md +++ b/openhands-agent-server/AGENTS.md @@ -123,7 +123,7 @@ feature map (`control-agent-server map coverage`). - `ConversationState` uses a synchronous `FIFOLock`. In async agent-server code, never do `with conversation._state` directly on the event loop when the conversation may be running. - WebSocket reconnects call `EventService.subscribe_to_events()` immediately; if initial state snapshot creation blocks on the state lock in async context, the whole FastAPI event loop can stop serving `/ready` and similar probes. - The same rule applies to metadata updates in `ConversationService.update_conversation()`: keep the locked mutation/snapshot semantics, but move the synchronous lock wait into a worker thread first. -- In async routes/services, move state-lock acquisition into `run_in_executor(...)` (or another worker-thread boundary) before awaiting network I/O. +- In async routes/services, move state-lock acquisition into `EventService.run_state_op(...)` before awaiting network I/O. It runs on the conversation's own executor: a run holds the lock for a whole tool step, and waiters on asyncio's default executor (`run_in_executor(None, ...)`, `asyncio.to_thread`) take the threads that event reads of every conversation need. ## REST API compatibility & deprecation policy diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 6942f67ac2c..ee6a501fdb6 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -1,4 +1,5 @@ import asyncio +import contextvars import importlib import json import logging @@ -7,6 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager, suppress from dataclasses import dataclass, field +from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any, cast from uuid import UUID, uuid4 @@ -2000,7 +2002,6 @@ async def update_conversation( if event_service is None: return False - loop = asyncio.get_running_loop() state = await event_service.get_state() update: dict[str, Any] = {"updated_at": utc_now()} if request.title is not None: @@ -2015,8 +2016,8 @@ async def update_conversation( if request.tags is not None: # Keep the persisted ConversationState update under the state lock so # autosave and state-change callbacks observe a consistent mutation. - state = await loop.run_in_executor( - None, _update_state_tags_sync, state, request.tags + state = await event_service.run_state_op( + _update_state_tags_sync, state, request.tags ) if budget_updated: # base_state.json owns the budget; the event service is loaded @@ -2030,8 +2031,8 @@ async def update_conversation( # Notify conversation webhooks about the updated conversation. Compose the # full-state snapshot under the state lock, but do the synchronous wait in a # worker thread so metadata updates cannot block the FastAPI event loop. - conversation_info = await loop.run_in_executor( - None, _compose_webhook_conversation_info_sync, event_service.stored, state + conversation_info = await event_service.run_state_op( + _compose_webhook_conversation_info_sync, event_service.stored, state ) await self._notify_conversation_webhooks(conversation_info) @@ -2146,12 +2147,15 @@ async def fork_conversation( raise ValueError(f"Conversation with id {fork_id} already exists") # fork() deep-copies events and state into ``fork_dir``. - fork_conv = await asyncio.to_thread( - source_conversation.fork, - conversation_id=fork_conv_id, - tags=tags, - reset_metrics=reset_metrics, - from_event_id=from_event_id, + fork_conv = await source_service.run_state_op( + partial( + contextvars.copy_context().run, + source_conversation.fork, + conversation_id=fork_conv_id, + tags=tags, + reset_metrics=reset_metrics, + from_event_id=from_event_id, + ) ) # Extract the persisted data, then discard the temporary conversation. fork_agent = cast(AgentBase, fork_conv.agent) diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index f1a6ef3c0fb..815ba3b2f38 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -2,14 +2,14 @@ import threading import time import traceback -from collections.abc import Mapping +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext, suppress from dataclasses import dataclass, field from datetime import datetime from functools import partial from pathlib import Path -from typing import cast +from typing import Any, cast from uuid import UUID, uuid4 from pydantic import ValidationError @@ -317,6 +317,12 @@ class EventService: _lease_task: asyncio.Task | None = field(default=None, init=False) _external_lease_renewal: bool = field(default=False, init=False) _run_executor: ThreadPoolExecutor | None = field(default=None, init=False) + # Runs the calls that acquire this conversation's state lock. A run holds + # that lock for a whole tool step, so on asyncio's shared default executor + # the waiters would take the threads that lock-free work needs (event reads + # of every conversation). Here they only queue behind their own + # conversation. + _state_executor: ThreadPoolExecutor | None = field(default=None, init=False) _run_semaphore: asyncio.Semaphore | None = field(default=None, init=False) # Permit for this conversation's current admission session. Held for the # whole chain of runs (initial run, any re-arm for stranded input, ACP @@ -343,6 +349,11 @@ class EventService: def conversation_dir(self): return self.conversations_dir / self.stored.id.hex + async def run_state_op[R](self, fn: Callable[..., R], *args: Any) -> R: + """Run ``fn``, which acquires the state lock, off the event loop.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor(self._state_executor, fn, *args) + async def load_meta(self): meta_file = self.conversation_dir / "meta.json" self.stored = StoredConversation.model_validate_json( @@ -528,7 +539,7 @@ def _update() -> None: if isinstance(agent, ACPAgent): agent.restart_for_updated_credentials(secrets) - await asyncio.to_thread(_update) + await self.run_state_op(_update) await self.save_meta( self.stored.model_copy( update={"secrets": {**self.stored.secrets, **secrets}} @@ -834,8 +845,7 @@ def _get_execution_status_sync(self) -> ConversationExecutionStatus: return state.execution_status async def _get_execution_status(self) -> ConversationExecutionStatus: - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._get_execution_status_sync) + return await self.run_state_op(self._get_execution_status_sync) def _mark_error_status_sync(self) -> None: """Force the conversation into ERROR status (idempotent backstop). @@ -894,8 +904,7 @@ def _create_initial_state_sync( return self._create_state_update_event_sync(), len(state.events) async def _create_state_update_event(self) -> ConversationStateUpdateEvent: - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._create_state_update_event_sync) + return await self.run_state_op(self._create_state_update_event_sync) def _event_matches_body(self, event: Event, body: str) -> bool: """Check if event's message content matches body filter (case-insensitive).""" @@ -961,13 +970,12 @@ async def send_message( if not _from_goal_loop: await self.stop_goal_loop() explicit_interrupt_generation = self._explicit_interrupt_generation - loop = asyncio.get_running_loop() send_message = ( partial(self._conversation.send_message, sender="goal") if _from_goal_loop else self._conversation.send_message ) - await loop.run_in_executor(None, send_message, message) + await self.run_state_op(send_message, message) if run: if self._explicit_interrupt_generation != explicit_interrupt_generation: return @@ -1071,10 +1079,7 @@ def _mark_running_acp_prompt_superseded_sync(self) -> tuple[bool, bool]: return (True, False) async def _mark_running_acp_prompt_superseded(self) -> tuple[bool, bool]: - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, self._mark_running_acp_prompt_superseded_sync - ) + return await self.run_state_op(self._mark_running_acp_prompt_superseded_sync) async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: subscriber_id = self._pub_sub.subscribe(subscriber) @@ -1084,9 +1089,8 @@ async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: # conversation's synchronous FIFOLock cannot block the server event loop. log_length: int | None = None if self._conversation: - loop = asyncio.get_running_loop() - state_update_event, log_length = await loop.run_in_executor( - None, self._create_initial_state_sync + state_update_event, log_length = await self.run_state_op( + self._create_initial_state_sync ) else: state_update_event = ConversationStateUpdateEvent( @@ -1161,7 +1165,7 @@ def locked_on_event(): # needs a done-callback: otherwise it is collected with an unread # exception, so a dropped stats or LLM-log event leaves no signal # beyond a GC-time warning. - future = main_loop.run_in_executor(None, locked_on_event) + future = main_loop.run_in_executor(self._state_executor, locked_on_event) # Attach from the loop thread. This helper is called from foreign # threads, and `add_done_callback` on an already-done future uses # plain `call_soon`, which neither is thread-safe nor wakes an idle @@ -1318,6 +1322,12 @@ def _prepare_workspace_dir(working_dir: Path) -> None: async def start(self): # Store the main event loop for cross-thread communication self._main_loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + # Not created on first use: _emit_event_from_thread submits to it from + # other threads. Sized like the default executor it replaces; threads + # start on demand. + self._state_executor = ThreadPoolExecutor( + thread_name_prefix="conversation-state" + ) # self.stored contains an Agent configuration we can instantiate self.conversation_dir.mkdir(parents=True, exist_ok=True) @@ -1553,9 +1563,7 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: # still active was left by a shutdown or crash mid-loop (close() skips # the status on purpose). Record it as interrupted so it reads as # stopped and stays resumable. - await asyncio.get_running_loop().run_in_executor( - None, self._reconcile_stale_goal_status - ) + await self.run_state_op(self._reconcile_stale_goal_status) # Publish initial state update await self._publish_state_update() @@ -1768,10 +1776,8 @@ async def _run_and_publish(): # ConversationRunError means run()/arun() already emitted its # own event, so skip it there to avoid duplicating the error. if not isinstance(exc, ConversationRunError): - await loop.run_in_executor( - None, self._publish_error_event_sync, exc - ) - await loop.run_in_executor(None, self._mark_error_status_sync) + await self.run_state_op(self._publish_error_event_sync, exc) + await self.run_state_op(self._mark_error_status_sync) finally: settle() # Wait for all pending events to be published via @@ -1934,7 +1940,7 @@ def _require_answered() -> None: if find_pending_ask(conversation._state.active_branch()) is not None: raise ValueError("pending_ask") - await asyncio.get_running_loop().run_in_executor(None, _require_answered) + await self.run_state_op(_require_answered) if self._closing: raise ServiceInactive() # Reserve capacity before spawning the loop so a full server refuses @@ -1969,7 +1975,6 @@ async def _run_goal_loop( conversation = self._conversation if conversation is None: return - loop = asyncio.get_running_loop() def _snapshot_and_judge() -> GoalStep: # Snapshot events under the conversation lock, then judge (an LLM @@ -2006,7 +2011,7 @@ def _persist() -> None: with conversation._state: conversation._on_event(event) - await loop.run_in_executor(None, _persist) + await self.run_state_op(_persist) try: await _emit_status(active=True, status="running") @@ -2039,7 +2044,7 @@ def _persist() -> None: # that breaks the agent out of any genuine loop). Only # PAUSED/ERROR (real stop signals) terminate the goal. logger.info("Goal loop continuing past stuck run") - step = await loop.run_in_executor(None, _snapshot_and_judge) + step = await self.run_state_op(_snapshot_and_judge) if isinstance(step, GoalAsked): # The agent asked the user (ask_user). A followup would # land over the question, so stop here, resumable; the @@ -2162,8 +2167,7 @@ async def resume_goal_loop( """ if not self._conversation or self._closing: raise ServiceInactive() - loop = asyncio.get_running_loop() - last = await loop.run_in_executor(None, self._last_goal_loop_status) + last = await self.run_state_op(self._last_goal_loop_status) if last is None or last.get("status") in ("complete", "capped"): raise ValueError("no_resumable_goal") if judge_llm is None: @@ -2228,10 +2232,8 @@ async def react( if not self._conversation: raise ServiceInactive() conversation = self._conversation - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, - partial(conversation.react, emoji, target_event_id, removed=removed), + return await self.run_state_op( + partial(conversation.react, emoji, target_event_id, removed=removed) ) async def answer_ask( @@ -2247,10 +2249,8 @@ async def answer_ask( """ if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - answer, message = await loop.run_in_executor( - None, - partial(self._conversation.answer_ask, ask_id, option_index, text), + answer, message = await self.run_state_op( + self._conversation.answer_ask, ask_id, option_index, text ) await self.send_message(message, run=True) return answer @@ -2259,18 +2259,14 @@ async def reject_pending_actions(self, reason: str): """Reject all pending actions and publish updated state.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.reject_pending_actions, reason - ) + await self.run_state_op(self._conversation.reject_pending_actions, reason) async def pause(self): if self._conversation: self._explicit_interrupt_generation += 1 self._rerun_requested = False self._acp_internal_rerun_requested = False - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.pause) + await self.run_state_op(self._conversation.pause) # Publish state update after pause to ensure stats are updated await self._publish_state_update() @@ -2306,8 +2302,7 @@ async def add_mcp_servers(self, servers: dict[str, MCPServer]) -> None: """Add modern MCP servers to the conversation.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.add_mcp_servers, servers) + await self.run_state_op(self._conversation.add_mcp_servers, servers) await self._publish_state_update() async def update_secrets(self, secrets: dict[str, SecretValue]): @@ -2324,17 +2319,13 @@ async def update_secrets(self, secrets: dict[str, SecretValue]): if CODEX_AUTH_SECRET_NAME in self.credential_bindings: secrets = dict(secrets) secrets.pop(CODEX_AUTH_SECRET_NAME, None) - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.update_secrets, secrets) + await self.run_state_op(self._conversation.update_secrets, secrets) async def set_confirmation_policy(self, policy: ConfirmationPolicyBase): """Set the confirmation policy for the conversation.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.set_confirmation_policy, policy - ) + await self.run_state_op(self._conversation.set_confirmation_policy, policy) async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None: """Set the conversation's cost ceiling, taking effect from its next step. @@ -2346,9 +2337,8 @@ async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None conversation = self._conversation if conversation is None: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, setattr, conversation, "max_budget_per_run", max_budget_per_run + await self.run_state_op( + setattr, conversation, "max_budget_per_run", max_budget_per_run ) async def set_security_analyzer( @@ -2357,17 +2347,15 @@ async def set_security_analyzer( """Set the security analyzer for the conversation.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.set_security_analyzer, security_analyzer + await self.run_state_op( + self._conversation.set_security_analyzer, security_analyzer ) async def load_plugin(self, plugin_ref: str) -> None: """Load a marketplace plugin into the active conversation.""" if self._conversation is None: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.load_plugin, plugin_ref) + await self.run_state_op(self._conversation.load_plugin, plugin_ref) async def switch_acp_model(self, model: str) -> None: """Switch the model on an ACP conversation. @@ -2388,8 +2376,7 @@ async def switch_acp_model(self, model: str) -> None: # a created-but-not-yet-run conversation, so a pre-first-run switch # is a normal 200 deferral, not an error. raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.switch_acp_model, model) + await self.run_state_op(self._conversation.switch_acp_model, model) async def switch_acp_effort(self, effort: str) -> None: """Switch the reasoning effort on an ACP conversation. @@ -2399,8 +2386,7 @@ async def switch_acp_effort(self, effort: str) -> None: """ if self._conversation is None: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.switch_acp_effort, effort) + await self.run_state_op(self._conversation.switch_acp_effort, effort) async def close_connections(self, code: int, reason: str) -> None: """Close the client sockets subscribed to this conversation.""" @@ -2438,9 +2424,8 @@ async def close(self): run_task = self._run_task if run_task is not None and not run_task.done(): if self._conversation is not None: - loop = asyncio.get_running_loop() try: - await loop.run_in_executor(None, self._conversation.pause) + await self.run_state_op(self._conversation.pause) except Exception: logger.warning( "Failed to pause conversation during close", exc_info=True @@ -2468,6 +2453,9 @@ async def close(self): loop = asyncio.get_running_loop() await loop.run_in_executor(None, self._conversation.close) self._conversation = None + if self._state_executor is not None: + self._state_executor.shutdown(wait=False) + self._state_executor = None self.credential_bindings = {} if self._lease is not None and self._lease_generation is not None: @@ -2509,8 +2497,7 @@ async def ask_agent(self, question: str) -> str: if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._conversation.ask_agent, question) + return await self.run_state_op(self._conversation.ask_agent, question) async def condense(self) -> None: """Force condensation of the conversation history. @@ -2534,8 +2521,7 @@ async def condense(self) -> None: f"{type(condenser).__name__} does not handle condensation requests" ) - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._conversation.condense) + return await self.run_state_op(self._conversation.condense) async def navigate_to(self, event_id: str | None) -> None: """Move the conversation HEAD to an existing event (in-place re-root). @@ -2548,10 +2534,7 @@ async def navigate_to(self, event_id: str | None) -> None: if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, self._conversation.navigate_to, event_id - ) + return await self.run_state_op(self._conversation.navigate_to, event_id) def _get_agent_final_response_sync(self) -> str: """Extract the agent's final response from the active branch. diff --git a/openhands-sdk/openhands/sdk/agent/parallel_executor.py b/openhands-sdk/openhands/sdk/agent/parallel_executor.py index c32e4714156..eb705ca1e71 100644 --- a/openhands-sdk/openhands/sdk/agent/parallel_executor.py +++ b/openhands-sdk/openhands/sdk/agent/parallel_executor.py @@ -159,16 +159,29 @@ def _resolve(ae: ActionEvent) -> ToolDefinition | None: return tools.get(ae.tool_name) if tools else None if len(action_events) == 1 or self._max_workers == 1: - return [ - await self._arun_safe( - action, - tool_runner, - _resolve(action), - cancel_token, - span_owner=span_owner, - ) - for action in action_events - ] + # Not asyncio's default executor: arun() holds the conversation + # state lock across this await, and callers waiting on that lock + # can take every default-executor thread. A tool queued behind + # them would never start, so the lock would never be released. + pool = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="aexecute_batch" + ) + try: + return [ + await self._arun_safe( + action, + tool_runner, + _resolve(action), + cancel_token, + pool, + span_owner, + ) + for action in action_events + ] + finally: + # No wait: an interrupt cancels the await above while the tool + # thread still runs, and waiting would block the event loop. + pool.shutdown(wait=False) with ThreadPoolExecutor( max_workers=self._max_workers, @@ -206,9 +219,8 @@ async def _arun_safe( locks are acquired on the worker thread, not the event-loop thread. - When *executor* is ``None`` the asyncio default pool is used - (suitable for one-off / single-action calls); batch callers - should pass a dedicated pool so concurrent tool calls don't + When *executor* is ``None`` the asyncio default pool is used; + batch callers should pass a dedicated pool so tool calls don't contend with other ``run_in_executor`` users on the shared default pool. diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 38daad6532a..65c98663663 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -2973,6 +2973,7 @@ async def test_update_conversation_tags_uses_state_lock( """Test that tag updates hold the ConversationState lock.""" mock_service = AsyncMock(spec=EventService) mock_service.stored = sample_stored_conversation + mock_service.run_state_op.side_effect = asyncio.to_thread _adopt_saved_meta(mock_service) mock_state = ConversationState( id=sample_stored_conversation.id, @@ -3008,6 +3009,7 @@ async def test_update_conversation_tags_wait_does_not_block_event_loop( """Waiting on the state lock must not stall unrelated async work.""" mock_service = AsyncMock(spec=EventService) mock_service.stored = sample_stored_conversation + mock_service.run_state_op.side_effect = asyncio.to_thread _adopt_saved_meta(mock_service) state = ConversationState( id=sample_stored_conversation.id, @@ -3096,6 +3098,7 @@ async def test_update_conversation_notifies_webhooks( # Create mock event service mock_service = AsyncMock(spec=EventService) mock_service.stored = sample_stored_conversation + mock_service.run_state_op.side_effect = asyncio.to_thread _adopt_saved_meta(mock_service) mock_state = ConversationState( id=sample_stored_conversation.id, @@ -3144,6 +3147,7 @@ async def test_update_acp_conversation_notifies_webhooks_with_acp_shape( ) mock_service = AsyncMock(spec=EventService) mock_service.stored = stored_conversation + mock_service.run_state_op.side_effect = asyncio.to_thread _adopt_saved_meta(mock_service) mock_state = ConversationState( id=stored_conversation.id, diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index b01fab373b0..8e270bbc7f0 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -7,6 +7,7 @@ import threading import time import zipfile +from concurrent.futures import ThreadPoolExecutor from contextlib import suppress from datetime import UTC, datetime from pathlib import Path @@ -4115,6 +4116,58 @@ def recording_check(working_dir: Path) -> None: assert threads[0] != loop_thread +@pytest.mark.asyncio +@pytest.mark.parametrize( + "wait_on_state_lock", + [ + pytest.param( + lambda service: service.subscribe_to_events(AsyncMock()), + id="websocket-subscribe", + ), + pytest.param( + lambda service: service.send_message( + Message(role="user", content=[TextContent(text="hi")]) + ), + id="send-message", + ), + pytest.param(lambda service: service.pause(), id="pause"), + ], +) +async def test_state_lock_waiters_leave_the_default_executor_free( + tmp_path: Path, wait_on_state_lock +) -> None: + """A run holds the state lock for a whole tool step. Requests waiting on it + must not take default-executor threads: lock-free work, such as the event + reads of every conversation, needs that pool (#6553).""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + service = EventService( + stored=_make_stored(tmp_path), agent=_sample_agent(), conversations_dir=tmp_path + ) + await service.start() + state = service.get_conversation()._state + held, release = threading.Event(), threading.Event() + + def hold_state_lock() -> None: # stands in for a run inside a tool step + with state: + held.set() + release.wait() + + holder = threading.Thread(target=hold_state_lock) + holder.start() + held.wait() + waiter = asyncio.create_task(wait_on_state_lock(service)) + try: + await asyncio.sleep(0.1) # let the request reach the lock + probe = loop.run_in_executor(None, lambda: "free") + assert await asyncio.wait_for(probe, timeout=2) == "free" + finally: + release.set() + holder.join() + await asyncio.wait_for(waiter, timeout=5) + await service.close() + + def test_concurrent_repo_checks_on_a_new_directory_all_succeed( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/sdk/agent/test_parallel_executor.py b/tests/sdk/agent/test_parallel_executor.py index 8436a04c5c8..8e30254cb05 100644 --- a/tests/sdk/agent/test_parallel_executor.py +++ b/tests/sdk/agent/test_parallel_executor.py @@ -1,7 +1,9 @@ """Tests for ParallelToolExecutor.""" +import asyncio import threading import time +from concurrent.futures import ThreadPoolExecutor from typing import Any from unittest.mock import MagicMock @@ -238,3 +240,25 @@ def outer_tool_runner(action: Any) -> list: ) assert results == [["inner-a", "inner-b"]] + + +async def test_async_sequential_batch_runs_with_the_default_executor_full(): + """arun() holds the state lock across the batch, and callers waiting on that + lock can take every default-executor thread. A tool queued behind them + would never start, so the lock would never be released (#6553).""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + release = threading.Event() + waiter = loop.run_in_executor(None, release.wait) + actions: list[Any] = [MagicMock(), MagicMock()] + event = MagicMock() + try: + results = await asyncio.wait_for( + ParallelToolExecutor().aexecute_batch(actions, lambda action: [event]), + timeout=5, + ) + finally: + release.set() + await waiter + + assert results == [[event], [event]]