From 8f9eab874b6c86780062b335f8e2f02e771c0792 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Thu, 12 Nov 2026 10:26:08 +0100 Subject: [PATCH] refactor(agent-server): reach stores through run_store and enable ruff ASYNC The persistence stores are synchronous and wait on a file lock. Async handlers called them on the event loop, or each picked its own asyncio.to_thread hop, so one store waiting on its lock stalled every request. - run_store(fn, *args, **kwargs) in openhands.agent_server.persistence is the one way async code reaches a store. It replaces 16 hand-placed store hops, 17 direct blocking store calls and 8 calls to sync helpers that did store I/O on the loop. - scripts/check_async_store_access.py (pre-commit) rejects store calls made directly in async code or hopped without run_store. - Ruff ASYNC rules are on; the 29 production violations are fixed, with a stated reason for each of the five noqa comments. Tests ignore ASYNC like they ignore ARG. Part of #6456 (step 3). --- .../programs/S21/acp_conversation.py | 3 +- .pre-commit-config.yaml | 8 + .../agent_server/agent_profiles_router.py | 5 +- .../openhands/agent_server/api.py | 4 +- .../agent_server/canvas_extensions/backend.py | 33 +- .../agent_server/conversation_router.py | 6 +- .../agent_server/conversation_service.py | 13 +- .../agent_server/credential_binding.py | 6 +- .../agent_server/docker_runtime/proxy.py | 10 +- .../agent_server/docker_runtime/routers.py | 77 +++-- .../openhands/agent_server/event_router.py | 10 +- .../openhands/agent_server/event_service.py | 16 +- .../experimental_features_router.py | 10 +- .../openhands/agent_server/file_router.py | 128 ++++--- .../agent_server/git_provider_service.py | 6 +- .../openhands/agent_server/init_router.py | 8 +- .../openhands/agent_server/mcp_oauth_store.py | 15 +- .../openhands/agent_server/openai/service.py | 11 +- .../agent_server/persistence/__init__.py | 3 + .../agent_server/persistence/async_store.py | 36 ++ .../openhands/agent_server/profiles_router.py | 5 +- .../openhands/agent_server/runtime_router.py | 15 +- .../openhands/agent_server/settings_router.py | 19 +- .../agent_server/telemetry/service.py | 4 +- .../openhands/agent_server/vscode_service.py | 3 +- .../agent_server/workspace_router.py | 5 +- .../agent_server/workspaces_router.py | 26 +- openhands-sdk/openhands/sdk/llm/llm.py | 4 +- .../remote/async_remote_workspace.py | 4 +- pyproject.toml | 4 +- scripts/check_async_store_access.py | 315 ++++++++++++++++++ .../agent_server/stress/test_slow_webhook.py | 2 +- .../test_profile_store_lock_responsiveness.py | 24 ++ tests/cross/test_check_async_store_access.py | 143 ++++++++ 34 files changed, 798 insertions(+), 183 deletions(-) create mode 100644 openhands-agent-server/openhands/agent_server/persistence/async_store.py create mode 100644 scripts/check_async_store_access.py create mode 100644 tests/cross/test_check_async_store_access.py diff --git a/.agents/skills/verify-sdk/programs/S21/acp_conversation.py b/.agents/skills/verify-sdk/programs/S21/acp_conversation.py index 403610d5221..2cd6bf97235 100644 --- a/.agents/skills/verify-sdk/programs/S21/acp_conversation.py +++ b/.agents/skills/verify-sdk/programs/S21/acp_conversation.py @@ -562,7 +562,8 @@ async def cancel_mid_prompt() -> str: conversation.send_message("[cancel] build everything") task = asyncio.create_task(conversation.arun()) deadline = time.monotonic() + 30 - while not journal("cancel_waiting") and time.monotonic() < deadline: + # The fake ACP server only writes a journal file; nothing to await. + while not journal("cancel_waiting") and time.monotonic() < deadline: # noqa: ASYNC110 await asyncio.sleep(0.1) await asyncio.sleep(0.3) task.cancel() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd95d4fb9d0..4692092fe47 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,3 +62,11 @@ repos: types: [python] pass_filenames: true always_run: false + - id: check-async-store-access + name: Keep blocking store I/O off the agent-server event loop + entry: uv + args: [run, python, scripts/check_async_store_access.py] + language: system + files: ^openhands-agent-server/.*\.py$ + pass_filenames: true + always_run: false diff --git a/openhands-agent-server/openhands/agent_server/agent_profiles_router.py b/openhands-agent-server/openhands/agent_server/agent_profiles_router.py index 03ca50f4af9..dc43d93c685 100644 --- a/openhands-agent-server/openhands/agent_server/agent_profiles_router.py +++ b/openhands-agent-server/openhands/agent_server/agent_profiles_router.py @@ -34,6 +34,7 @@ get_agent_profile_store, get_llm_profile_store, get_settings_store, + run_store, ) from openhands.agent_server.profiles_router import MAX_PROFILES, _has_api_key from openhands.sdk.launch import preview_launch @@ -605,7 +606,7 @@ async def materialize_agent_profile( try: with store_errors(): # Off the event loop: the store lock wait blocks (up to 30 s). - profile = await asyncio.to_thread(store.load, name) + profile = await run_store(store.load, name) except FileNotFoundError: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -613,7 +614,7 @@ async def materialize_agent_profile( ) config = get_config(request) - settings = get_settings_store(config).load() or PersistedSettings() + settings = await run_store(get_settings_store(config).load) or PersistedSettings() context = settings.agent_settings.agent_context return await asyncio.to_thread( lambda: preview_launch( diff --git a/openhands-agent-server/openhands/agent_server/api.py b/openhands-agent-server/openhands/agent_server/api.py index a10e5497baa..486439f7ec5 100644 --- a/openhands-agent-server/openhands/agent_server/api.py +++ b/openhands-agent-server/openhands/agent_server/api.py @@ -86,7 +86,7 @@ check_openai_api_key, openai_router, ) -from openhands.agent_server.persistence import reencrypt_stale_stores +from openhands.agent_server.persistence import reencrypt_stale_stores, run_store from openhands.agent_server.plugins_router import plugins_router from openhands.agent_server.profiles_router import profiles_router from openhands.agent_server.provider_connections_router import ( @@ -419,7 +419,7 @@ async def stop_tool_preload_service(): # Non-deferred (legacy) path: build and enter the conversation # service as part of the lifespan, exactly as before. - await asyncio.to_thread(reencrypt_stale_stores, config) + await run_store(reencrypt_stale_stores, config) # # A key that came from the config file is visible only in ``config``; # in-process callers of this server read it from the environment. diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py index a387fe0b1a2..b6330590a56 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py @@ -452,7 +452,7 @@ async def prepare(self, name: str, revision: str) -> BackendStatus: if ( prepared is not None and prepared.revision == revision - and Path(prepared.artifact_dir).is_dir() + and await asyncio.to_thread(Path(prepared.artifact_dir).is_dir) ): return self._status_unlocked(name) @@ -786,19 +786,20 @@ async def start(self, name: str, revision: str) -> BackendStatus: if prepared is None or prepared.revision != revision: raise RuntimeError("backend revision must be prepared before start") artifact_dir = Path(prepared.artifact_dir) - if not artifact_dir.is_dir(): + if not await asyncio.to_thread(artifact_dir.is_dir): raise RuntimeError("prepared backend artifact is missing") parts = self._backend_parts(name) assert parts is not None backend, artifact, current_platform = parts - expected_artifact_dir = ( - self.artifacts_dir / name / prepared.artifact_sha256 - ).resolve() + expected_artifact_dir = await asyncio.to_thread( + (self.artifacts_dir / name / prepared.artifact_sha256).resolve + ) if ( prepared.platform != current_platform or prepared.artifact_sha256 != artifact.sha256 - or artifact_dir.resolve() != expected_artifact_dir + or await asyncio.to_thread(artifact_dir.resolve) + != expected_artifact_dir ): raise RuntimeError( "prepared backend metadata does not match manifest" @@ -912,8 +913,12 @@ def _runtime_group_alive(self, runtime: _Runtime) -> bool: process.returncode is None or self._group_alive(process.pid) ) - async def _wait_for_group_exit(self, pgid: int, timeout: float) -> bool: - deadline = time.monotonic() + timeout + async def _wait_for_group_exit(self, pgid: int) -> bool: + """Whether the group exits within ``_STOP_TIMEOUT_SECONDS``. + + Nothing signals a process group's exit, so this polls. + """ + deadline = time.monotonic() + _STOP_TIMEOUT_SECONDS while True: if not self._group_alive(pgid): return True @@ -932,14 +937,14 @@ async def _terminate_runtime(self, runtime: _Runtime) -> None: # on the leader's exit code would leak those descendants. if process.returncode is None: self._signal_group(pgid, signal.SIGTERM) - if not await self._wait_for_group_exit(pgid, _STOP_TIMEOUT_SECONDS): + if not await self._wait_for_group_exit(pgid): self._signal_group(pgid, signal.SIGKILL) - await self._wait_for_group_exit(pgid, _STOP_TIMEOUT_SECONDS) + await self._wait_for_group_exit(pgid) elif self._group_alive(pgid): # With no leader left to shut down cleanly, its descendants are # orphaned; do not let one hold status or restart for the grace period. self._signal_group(pgid, signal.SIGKILL) - await self._wait_for_group_exit(pgid, _STOP_TIMEOUT_SECONDS) + await self._wait_for_group_exit(pgid) if process.returncode is None: await process.wait() await self._finish_runtime(runtime) @@ -998,11 +1003,9 @@ async def reap_orphans(self) -> None: continue # Its server is still running and owns it. if self._is_recorded_backend(record): self._signal_group(record.pgid, signal.SIGTERM) - if not await self._wait_for_group_exit( - record.pgid, _STOP_TIMEOUT_SECONDS - ): + if not await self._wait_for_group_exit(record.pgid): self._signal_group(record.pgid, signal.SIGKILL) - await self._wait_for_group_exit(record.pgid, _STOP_TIMEOUT_SECONDS) + await self._wait_for_group_exit(record.pgid) path.unlink(missing_ok=True) def has_running_backends(self) -> bool: diff --git a/openhands-agent-server/openhands/agent_server/conversation_router.py b/openhands-agent-server/openhands/agent_server/conversation_router.py index 42bfc98a07e..cc6b0d65430 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_router.py +++ b/openhands-agent-server/openhands/agent_server/conversation_router.py @@ -54,7 +54,7 @@ UpdateSecretsRequest, trim_conversation_response_skills, ) -from openhands.agent_server.persistence import get_llm_profile_store +from openhands.agent_server.persistence import get_llm_profile_store, run_store from openhands.sdk import LLM from openhands.sdk.conversation.state import ConversationExecutionStatus from openhands.sdk.conversation.types import ( @@ -656,7 +656,9 @@ async def switch_conversation_llm( # is replaced. Inline-key configs without a provider_connection_id are # byte-identical to the old path. with store_errors(): - llm = get_llm_profile_store().resolve_provider_connection(llm, cipher=cipher) + llm = await run_store( + get_llm_profile_store().resolve_provider_connection, llm, cipher=cipher + ) try: conversation.switch_llm(llm) except ValueError as e: diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 39003f51c78..cc495b20cfb 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -51,7 +51,7 @@ StoredConversation, UpdateConversationRequest, ) -from openhands.agent_server.persistence import FileSecretsStore +from openhands.agent_server.persistence import FileSecretsStore, run_store from openhands.agent_server.pub_sub import Subscriber from openhands.agent_server.server_details_router import update_last_execution_time from openhands.agent_server.telemetry import ( @@ -794,10 +794,7 @@ def _is_codex_agent(agent: AgentBase | None) -> bool: async def _has_local_codex_credential(self) -> bool: if self.secrets_store is None: return False - value = await asyncio.to_thread( - self.secrets_store.get_secret, - CODEX_AUTH_SECRET_NAME, - ) + value = await run_store(self.secrets_store.get_secret, CODEX_AUTH_SECRET_NAME) return value is not None async def _resolve_credential_bindings( @@ -1575,9 +1572,7 @@ async def _create_conversation( # now that every launch reads it, a bad file mode must not take down # request shapes that need nothing from settings. try: - settings = await asyncio.to_thread( - lambda: get_settings_store().load() or PersistedSettings() - ) + settings = await run_store(get_settings_store().load) or PersistedSettings() except (PermissionError, OSError): logger.warning( "Cannot read settings; starting without the stored agent preferences", @@ -2763,7 +2758,7 @@ async def __call__(self, event: Event) -> None: # truncation. This keeps auto-titling non-breaking for consumers who # don't configure title_llm_profile. conversation = self.service._conversation - title_llm = self._load_title_llm() + title_llm = await run_store(self._load_title_llm) if title_llm is None: title_llm = conversation.agent.llm if conversation else None diff --git a/openhands-agent-server/openhands/agent_server/credential_binding.py b/openhands-agent-server/openhands/agent_server/credential_binding.py index 3674e52b3eb..1c912f18f1a 100644 --- a/openhands-agent-server/openhands/agent_server/credential_binding.py +++ b/openhands-agent-server/openhands/agent_server/credential_binding.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from openhands.agent_server.event_service import CredentialBindingActivationTooLate -from openhands.agent_server.persistence import FileSecretsStore +from openhands.agent_server.persistence import FileSecretsStore, run_store from openhands.sdk.agent.acp_file_credentials import supports_file_credential_binding from openhands.sdk.credential import ( CredentialAuthorizationRejected, @@ -48,7 +48,7 @@ class LocalVersionedCredentialBinding: async def load(self) -> ResolvedCredential: try: - value, version = await asyncio.to_thread( + value, version = await run_store( self.store.load_versioned_secret, self.secret_name, ) @@ -62,7 +62,7 @@ async def load(self) -> ResolvedCredential: async def replace(self, expected_version: str, value: str) -> str: try: - return await asyncio.to_thread( + return await run_store( self.store.replace_versioned_secret, self.secret_name, expected_version, diff --git a/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py b/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py index 2c23ee5ec4c..b848213ad9a 100644 --- a/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py +++ b/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py @@ -94,7 +94,7 @@ async def proxy_http( workspace: ProxyTarget, *, upstream_path: str, - timeout: float | None = None, + read_timeout: float | None = None, body: bytes | None = None, on_close: Callable[[], Any] | None = None, reject_redirects: bool = False, @@ -108,8 +108,8 @@ async def proxy_http( agent-server to forward to. Typically the same path the outer server received, since the inner agent-server exposes the same API surface. - timeout: Per-request timeout in seconds. ``None`` (the default) means - no read timeout — conversation event streams can be long-lived. + read_timeout: Upstream read timeout in seconds. ``None`` (the default) + means no read timeout — conversation event streams can be long-lived. body: Replacement request body. By default the incoming body is streamed. on_close: Awaited or invoked once the streamed response is fully consumed, client disconnects, or connection fails. Callers use this to release @@ -157,7 +157,9 @@ async def _request_body() -> AsyncIterator[bytes]: try: client = await stack.enter_async_context( httpx.AsyncClient( - timeout=httpx.Timeout(connect=10.0, read=timeout, write=30.0, pool=10.0) + timeout=httpx.Timeout( + connect=10.0, read=read_timeout, write=30.0, pool=10.0 + ) ) ) upstream = await stack.enter_async_context( diff --git a/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py b/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py index 2fe72a865f1..407f0493f36 100644 --- a/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py +++ b/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py @@ -40,7 +40,11 @@ StartConversationRequest, UpdateSecretsRequest, ) -from openhands.agent_server.persistence import PersistedSettings, get_settings_store +from openhands.agent_server.persistence import ( + PersistedSettings, + get_settings_store, + run_store, +) from openhands.agent_server.utils import safe_rmtree from openhands.sdk.conversation.request import ConversationContainerOptions from openhands.sdk.git.exceptions import GitCommandError, GitRepositoryError @@ -129,6 +133,32 @@ async def detach() -> None: ) +def _worktree_git_paths(workspace_path: Path) -> dict[str, Path]: + """The repository and git common dir a worktree start records, if any.""" + try: + validate_git_repository(workspace_path) + repo_root = Path( + run_git_command( + ["git", "--no-pager", "rev-parse", "--show-toplevel"], + workspace_path, + ) + ).resolve() + git_common_dir = Path( + run_git_command( + ["git", "--no-pager", "rev-parse", "--git-common-dir"], + workspace_path, + ) + ) + except (GitCommandError, GitRepositoryError): + return {} + if not git_common_dir.is_absolute(): + git_common_dir = workspace_path / git_common_dir + return { + "worktree_repository_path": repo_root, + "worktree_git_common_dir_path": git_common_dir.resolve(), + } + + @docker_conversation_router.post("") async def start_conversation( request: Request, @@ -194,33 +224,14 @@ async def start_docker_conversation( payload, launched, options, worktree = await _prepare_forward( body, registry, existing=existing ) - identity = registry.provisioning.create(conversation_id, host_workspace) + identity = await run_store( + registry.provisioning.create, conversation_id, host_workspace + ) updates: dict[str, Any] = {} if worktree: - try: - validate_git_repository(identity.workspace_path) - repo_root = Path( - run_git_command( - ["git", "--no-pager", "rev-parse", "--show-toplevel"], - identity.workspace_path, - ) - ).resolve() - git_common_dir = Path( - run_git_command( - ["git", "--no-pager", "rev-parse", "--git-common-dir"], - identity.workspace_path, - ) - ) - if not git_common_dir.is_absolute(): - git_common_dir = identity.workspace_path / git_common_dir - git_common_dir = git_common_dir.resolve() - except (GitCommandError, GitRepositoryError): - pass - else: - updates.update( - worktree_repository_path=repo_root, - worktree_git_common_dir_path=git_common_dir, - ) + updates.update( + await asyncio.to_thread(_worktree_git_paths, identity.workspace_path) + ) if launched is not None and identity.launched_agent_profile is None: updates["launched_agent_profile"] = launched # Fixed once the container accepted the conversation; until then a @@ -229,7 +240,7 @@ async def start_docker_conversation( updates["container"] = options if updates: identity = identity.model_copy(update=updates) - registry.provisioning.save(identity) + await run_store(registry.provisioning.save, identity) container = await registry.get_or_create(conversation_id) release = registry.acquire_lease(conversation_id) forwarded = await asyncio.to_thread(payload, identity.cipher) @@ -290,7 +301,7 @@ async def _prepare_forward( context={"cipher": config.cipher} if body.get("secrets_encrypted") else None, ) try: - settings = await asyncio.to_thread(get_settings_store(config).load) + settings = await run_store(get_settings_store(config).load) except (OSError, PermissionError): settings = None settings = settings or PersistedSettings() @@ -335,7 +346,7 @@ async def runtime_credentials(conversation_id: UUID, request: Request) -> JSONRe registry = get_registry(request) if not registry.conversation_dir(conversation_id).joinpath("meta.json").is_file(): raise HTTPException(404, "Conversation not found") - identity = registry.provisioning.load(conversation_id) + identity = await run_store(registry.provisioning.load, conversation_id) return JSONResponse({"session_api_key": identity.api_key.get_secret_value()}) @@ -390,9 +401,10 @@ async def delete_conversation(conversation_id: UUID, request: Request) -> Respon logger.exception( "Could not clean up conversation worktree %s", conversation_id ) - await asyncio.to_thread( - safe_rmtree, registry.provisioning.runtime_dir(conversation_id) + runtime_dir = await run_store( + registry.provisioning.runtime_dir, conversation_id ) + await asyncio.to_thread(safe_rmtree, runtime_dir) await asyncio.to_thread(safe_rmtree, registry.conversation_dir(conversation_id)) registry.provisioning.manifest_path(conversation_id).unlink(missing_ok=True) registry.archived_marker(conversation_id).unlink(missing_ok=True) @@ -443,7 +455,8 @@ async def proxy_conversation( update = UpdateSecretsRequest.model_validate_json(await request.body()) except ValueError as exc: raise HTTPException(422, "Invalid secrets payload") from exc - profile = registry.provisioning.load(conversation_id).launched_agent_profile + identity = await run_store(registry.provisioning.load, conversation_id) + profile = identity.launched_agent_profile secrets = update.secrets if profile is not None: secrets = { diff --git a/openhands-agent-server/openhands/agent_server/event_router.py b/openhands-agent-server/openhands/agent_server/event_router.py index dab297962e7..1e676428057 100644 --- a/openhands-agent-server/openhands/agent_server/event_router.py +++ b/openhands-agent-server/openhands/agent_server/event_router.py @@ -39,7 +39,11 @@ Success, ) from openhands.agent_server.openai.usage import token_usage_delta -from openhands.agent_server.persistence import PersistedSettings, get_settings_store +from openhands.agent_server.persistence import ( + PersistedSettings, + get_settings_store, + run_store, +) from openhands.sdk import Message from openhands.sdk.conversation.state import ConversationExecutionStatus from openhands.sdk.event import Event, ReactionEvent @@ -367,8 +371,8 @@ async def respond_to_confirmation( async def require_reactions_enabled(config: Config) -> None: """Refuse with 409 unless the user has conversation reactions on.""" try: - settings = await asyncio.to_thread( - lambda: get_settings_store(config).load() or PersistedSettings() + settings = ( + await run_store(get_settings_store(config).load) or PersistedSettings() ) except (PermissionError, OSError): raise HTTPException( diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index 39b5487e1bb..605598dec7c 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1238,8 +1238,8 @@ def stats_callback() -> None: llm.telemetry.set_stats_update_callback(stats_callback) @staticmethod - def _ensure_workspace_is_git_repo(working_dir: Path) -> None: - """Initialize the workspace as a git repo if it isn't already one. + def _prepare_workspace_dir(working_dir: Path) -> None: + """Create the workspace and make it a git repo if it isn't already one. The /api/git/changes endpoint expects a real repository to compute changes against; without this, agent-created files never appear in @@ -1247,6 +1247,7 @@ def _ensure_workspace_is_git_repo(working_dir: Path) -> None: are handled by `get_valid_ref()` via GIT_EMPTY_TREE_HASH, and untracked files surface through `git ls-files --others`. """ + working_dir.mkdir(parents=True, exist_ok=True) try: validate_git_repository(working_dir) return # already a repo @@ -1284,9 +1285,9 @@ async def start(self): await self._scrub_persisted_credentials() workspace = self.stored.workspace assert isinstance(workspace, LocalWorkspace) - working_dir = Path(workspace.working_dir) - working_dir.mkdir(parents=True, exist_ok=True) - self._ensure_workspace_is_git_repo(working_dir) + await asyncio.to_thread( + self._prepare_workspace_dir, Path(workspace.working_dir) + ) # base_state.json is the single source of truth for the agent. On resume # (base_state exists) pass ``agent=None`` so LocalConversation keeps the # persisted agent. On a new conversation the creating caller supplied the @@ -1801,8 +1802,11 @@ async def _run_and_publish(): # coroutine enters its own try/finally. self._run_task.add_done_callback(settle) + # ``timeout`` bounds only the wait for the run task, not the status read + # after it, which a caller-side ``asyncio.timeout`` would also cut off. async def wait_for_run_completion( - self, timeout: float | None = None + self, + timeout: float | None = None, # noqa: ASYNC109 ) -> ConversationExecutionStatus: """Wait for the active run task without cancelling it on timeout.""" run_task = self._run_task diff --git a/openhands-agent-server/openhands/agent_server/experimental_features_router.py b/openhands-agent-server/openhands/agent_server/experimental_features_router.py index b88b0cd167a..ab485b34afe 100644 --- a/openhands-agent-server/openhands/agent_server/experimental_features_router.py +++ b/openhands-agent-server/openhands/agent_server/experimental_features_router.py @@ -1,7 +1,5 @@ """Routes for user-facing experimental features (see :mod:`.experimental_features`).""" -import asyncio - from fastapi import APIRouter, HTTPException, Request, status from openhands.agent_server._secrets_exposure import get_config @@ -12,7 +10,11 @@ find_experimental_feature, is_enabled, ) -from openhands.agent_server.persistence import PersistedSettings, get_settings_store +from openhands.agent_server.persistence import ( + PersistedSettings, + get_settings_store, + run_store, +) experimental_features_router = APIRouter( @@ -24,7 +26,7 @@ async def list_experimental_features(request: Request) -> list[ExperimentalFeature]: """List the features the user can turn on or off, and whether each is on.""" store = get_settings_store(get_config(request)) - settings = await asyncio.to_thread(lambda: store.load() or PersistedSettings()) + settings = await run_store(store.load) or PersistedSettings() return experimental_feature_flags(settings.experimental_features) diff --git a/openhands-agent-server/openhands/agent_server/file_router.py b/openhands-agent-server/openhands/agent_server/file_router.py index 65431e8afb8..8795d288953 100644 --- a/openhands-agent-server/openhands/agent_server/file_router.py +++ b/openhands-agent-server/openhands/agent_server/file_router.py @@ -99,18 +99,20 @@ async def _upload_file(path: str, file: UploadFile) -> FileUploadResult: detail="Path must be absolute", ) - # Ensure target directory exists - target_path.parent.mkdir(parents=True, exist_ok=True) + await asyncio.to_thread(target_path.parent.mkdir, parents=True, exist_ok=True) # Stream the file to disk to avoid memory issues with large files. - # Offload writes to a worker thread so slow storage (NFS, FUSE, - # encrypted FS) cannot starve the event loop for the upload's + # Every filesystem call runs on a worker thread so slow storage (NFS, + # FUSE, encrypted FS) cannot starve the event loop for the upload's # duration. file_size = 0 - with open(target_path, "wb") as f: + f = await asyncio.to_thread(target_path.open, "wb") + try: while chunk := await file.read(8192): # Read in 8KB chunks await asyncio.to_thread(f.write, chunk) file_size += len(chunk) + finally: + await asyncio.to_thread(f.close) logger.info(f"Uploaded file to {target_path}") return FileUploadResult(file_size=file_size) @@ -139,7 +141,7 @@ async def _upload_file(path: str, file: UploadFile) -> FileUploadResult: ) -async def _download_file(path: str) -> FileResponse: +def _download_file(path: str) -> FileResponse: """Internal helper to download a file from the workspace.""" update_last_execution_time() logger.info(f"Downloading file: {path}") @@ -252,7 +254,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: try: await super().__call__(scope, receive, send) finally: - Path(self.path).unlink(missing_ok=True) + await asyncio.to_thread(Path(self.path).unlink, missing_ok=True) ArchiveFormat = Literal["git-delta", "tar.gz"] @@ -332,6 +334,58 @@ def _is_sensitive_git_internal(rel: PurePosixPath) -> bool: _ARCHIVE_DESCENT_SKIP = frozenset({"node_modules", "venv", "site-packages", "vendor"}) +def _existing_directory(target: Path) -> Path: + """``target`` resolved, or the 404/400 for a missing path or a non-directory.""" + if not target.exists(): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Directory not found", + ) + if not target.is_dir(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Path is not a directory", + ) + return target.resolve() + + +def _archive_scratch_file(target: Path, suffix: str) -> Path: + """Create the empty file an archive of ``target`` is written to.""" + # Build the archive on the same volume as the workspace (its parent dir, so + # it is never included in itself) rather than the system temp dir: a large + # tar.gz on a tmpfs /tmp would defeat the stream-to-disk OOM fix, and on a + # small container root it could trip the pod's ephemeral-storage limit. Fall + # back to the default temp location if the parent is not usable. + scratch_dir: str | None = None + parent = target.parent + if parent != target and os.access(parent, os.W_OK): + scratch_dir = str(parent) + fd, tmp_name = tempfile.mkstemp(suffix=suffix, dir=scratch_dir) + os.close(fd) + return Path(tmp_name) + + +def _build_archive( + archive_format: ArchiveFormat, + repo_root: Path, + base_ref: str | None, + output_path: Path, + excludes: list[str], +) -> str: + """Write the archive to ``output_path``; the git-delta base commit or ``""``. + + ``output_path`` is removed when building fails. + """ + try: + if archive_format == "git-delta": + return _create_git_delta(repo_root, base_ref, output_path, excludes) + _create_tar_gz_archive(repo_root, output_path, excludes) + return "" + except Exception: + output_path.unlink(missing_ok=True) + raise + + def _resolve_git_repo_root(target: Path, max_depth: int = 3) -> Path: """Resolve the git work-tree to archive under ``target``. @@ -751,11 +805,11 @@ async def upload_file_query( @file_router.get( "/download", responses=_FILE_DOWNLOAD_RESPONSES, response_class=FileResponse ) -async def download_file_query( +def download_file_query( path: Annotated[str, Query(description="Absolute file path")], ) -> FileResponse: """Download a file from the workspace using query parameter (preferred method).""" - return await _download_file(path) + return _download_file(path) @file_router.post("/create_directory") @@ -876,7 +930,7 @@ async def get_home_directory( @file_discovery_router.get("/search_subdirs") -async def search_subdirs( +def search_subdirs( path: Annotated[ str, Query(description="Absolute directory path to list subdirectories of"), @@ -1010,7 +1064,7 @@ async def download_trajectory( try: await zip_task finally: - temp_file.unlink(missing_ok=True) + await asyncio.to_thread(temp_file.unlink, missing_ok=True) raise return _TemporaryFileResponse( path=temp_file, @@ -1101,80 +1155,50 @@ async def archive_directory( status_code=status.HTTP_400_BAD_REQUEST, detail="Path must be absolute", ) - if not target.exists(): - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Directory not found", - ) - if not target.is_dir(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Path is not a directory", - ) + target = await asyncio.to_thread(_existing_directory, target) if base_ref is not None and base_ref.startswith("-"): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="base_ref must not start with '-'", ) - target = target.resolve() effective_excludes = ( list(_DEFAULT_ARCHIVE_EXCLUDES) if use_default_excludes else [] ) + (exclude or []) - # Build the archive on the same volume as the workspace (its parent dir, so - # it is never included in itself) rather than the system temp dir: a large - # tar.gz on a tmpfs /tmp would defeat the stream-to-disk OOM fix, and on a - # small container root it could trip the pod's ephemeral-storage limit. Fall - # back to the default temp location if the parent is not usable. - scratch_dir: str | None = None - parent = target.parent - if parent != target and os.access(parent, os.W_OK): - scratch_dir = str(parent) - fd, tmp_name = tempfile.mkstemp( - suffix=_ARCHIVE_SUFFIX[archive_format], dir=scratch_dir + output_path = await asyncio.to_thread( + _archive_scratch_file, target, _ARCHIVE_SUFFIX[archive_format] ) - os.close(fd) - output_path = Path(tmp_name) # The caller passes the workspace base, but a repo-backed conversation clones # into a subdirectory; resolve to the actual repo so both formats root the # archive at the repo (consistent paths across git-delta, tar.gz, and the # initial snapshot) and git-delta does not 400 on the non-repo parent. repo_root = await asyncio.to_thread(_resolve_git_repo_root, target) - base_commit = "" try: - if archive_format == "git-delta": - base_commit = await asyncio.to_thread( - _create_git_delta, - repo_root, - base_ref, - output_path, - effective_excludes, - ) - else: - await asyncio.to_thread( - _create_tar_gz_archive, repo_root, output_path, effective_excludes - ) + base_commit = await asyncio.to_thread( + _build_archive, + archive_format, + repo_root, + base_ref, + output_path, + effective_excludes, + ) except GitRepositoryError as e: - output_path.unlink(missing_ok=True) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Not a git repository: {e}", ) except ValueError as e: - output_path.unlink(missing_ok=True) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=str(e), ) except GitCommandError as e: - output_path.unlink(missing_ok=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to generate git delta: {e}", ) except Exception as e: - output_path.unlink(missing_ok=True) logger.error(f"Failed to archive {target}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/openhands-agent-server/openhands/agent_server/git_provider_service.py b/openhands-agent-server/openhands/agent_server/git_provider_service.py index 1e7cd9178ef..232ddd0fcc8 100644 --- a/openhands-agent-server/openhands/agent_server/git_provider_service.py +++ b/openhands-agent-server/openhands/agent_server/git_provider_service.py @@ -9,7 +9,7 @@ import httpx from openhands.agent_server.config import Config -from openhands.agent_server.persistence import get_secrets_store +from openhands.agent_server.persistence import get_secrets_store, run_store from openhands.sdk.git.models import ( GitProviderRepository, GitProviderRepositoryPage, @@ -88,7 +88,7 @@ async def search_provider_repositories( f"Repository discovery is not supported for provider '{provider.value}'" ) - token = _resolve_provider_token(config, provider) + token = await run_store(_resolve_provider_token, config, provider) if token is None: return GitProviderRepositoryPage( items=[], next_page_id=None, missing_token=True @@ -122,7 +122,7 @@ async def get_provider_pull_request( raise UnsupportedGitProviderError( f"Pull requests are not supported for provider '{provider.value}'" ) - token = _resolve_provider_token(config, provider) + token = await run_store(_resolve_provider_token, config, provider) if token is None: raise MissingGitProviderTokenError(f"No {provider.value} token is configured") diff --git a/openhands-agent-server/openhands/agent_server/init_router.py b/openhands-agent-server/openhands/agent_server/init_router.py index e8636233e1b..b5eef520662 100644 --- a/openhands-agent-server/openhands/agent_server/init_router.py +++ b/openhands-agent-server/openhands/agent_server/init_router.py @@ -43,7 +43,11 @@ create_conversation_registry, ) from openhands.agent_server.conversation_service import ConversationService -from openhands.agent_server.persistence import reencrypt_stale_stores, reset_stores +from openhands.agent_server.persistence import ( + reencrypt_stale_stores, + reset_stores, + run_store, +) from openhands.agent_server.server_details_router import mark_initialization_complete from openhands.agent_server.telemetry import ( build_telemetry_sink, @@ -292,7 +296,7 @@ async def initialize(self, req: InitRequest) -> InitStatus: # an unpaired start and letting a retry emit a second one. await shutdown_telemetry_sink() reset_stores() - await asyncio.to_thread(reencrypt_stale_stores, new_config) + await run_store(reencrypt_stale_stores, new_config) self._app.state.telemetry_sink = await build_telemetry_sink(new_config) # Reset the module-level singleton so other call sites that go diff --git a/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py b/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py index e8a033479d5..49d14e4bed8 100644 --- a/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py +++ b/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py @@ -8,7 +8,6 @@ from __future__ import annotations -import asyncio import copy import threading from collections.abc import Mapping, Sequence @@ -18,7 +17,11 @@ import httpx from openhands.agent_server.config import Config, WebhookSpec -from openhands.agent_server.persistence import PersistedSettings, get_settings_store +from openhands.agent_server.persistence import ( + PersistedSettings, + get_settings_store, + run_store, +) from openhands.sdk.logger import get_logger from openhands.sdk.mcp.client import MCPClient from openhands.sdk.mcp.config import ( @@ -166,13 +169,13 @@ def _get_entry_sync( async def get( self, key: str, *, collection: str | None = None ) -> dict[str, Any] | None: - value, _ = await asyncio.to_thread(self._get_entry_sync, key, collection) + value, _ = await run_store(self._get_entry_sync, key, collection) return value async def ttl( self, key: str, *, collection: str | None = None ) -> tuple[dict[str, Any] | None, float | None]: - return await asyncio.to_thread(self._get_entry_sync, key, collection) + return await run_store(self._get_entry_sync, key, collection) def _put_sync( self, @@ -284,7 +287,7 @@ async def put( collection: str | None = None, ttl: SupportsFloat | None = None, ) -> None: - await asyncio.to_thread( + await run_store( self._put_sync, key, value, @@ -334,7 +337,7 @@ def apply_update(settings: PersistedSettings) -> PersistedSettings: return deleted async def delete(self, key: str, *, collection: str | None = None) -> bool: - return await asyncio.to_thread(self._delete_sync, key, collection) + return await run_store(self._delete_sync, key, collection) async def get_many( self, keys: Sequence[str], *, collection: str | None = None diff --git a/openhands-agent-server/openhands/agent_server/openai/service.py b/openhands-agent-server/openhands/agent_server/openai/service.py index 23aea5eb71d..aefb894c9e7 100644 --- a/openhands-agent-server/openhands/agent_server/openai/service.py +++ b/openhands-agent-server/openhands/agent_server/openai/service.py @@ -47,6 +47,7 @@ PersistedSettings, get_llm_profile_store, get_settings_store, + run_store, ) from openhands.sdk import Message from openhands.sdk.conversation.request import ( @@ -530,7 +531,7 @@ def iter_openai_chat_completion_sse( async def list_openai_models() -> OpenAIModelListResponse: try: - profiles = await asyncio.to_thread(get_llm_profile_store().list_summaries) + profiles = await run_store(get_llm_profile_store().list_summaries) except TimeoutError: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -759,7 +760,9 @@ async def run_chat_completion( detail="Streaming chat completions are not supported yet", ) - start_request = _conversation_request( + # Reads the profile and settings stores. + start_request = await run_store( + _conversation_request, request=request, config=config, conversation_id=reusable_conversation_id, @@ -828,7 +831,9 @@ async def run_response( ) user_content = _response_user_content(request) - start_request = _response_conversation_request( + # Reads the profile and settings stores. + start_request = await run_store( + _response_conversation_request, request=request, user_content=user_content, config=config, diff --git a/openhands-agent-server/openhands/agent_server/persistence/__init__.py b/openhands-agent-server/openhands/agent_server/persistence/__init__.py index 30805245cd9..866f6fbe8d5 100644 --- a/openhands-agent-server/openhands/agent_server/persistence/__init__.py +++ b/openhands-agent-server/openhands/agent_server/persistence/__init__.py @@ -6,6 +6,7 @@ See: openhands.sdk.settings.api_models """ +from openhands.agent_server.persistence.async_store import run_store from openhands.agent_server.persistence.models import ( PERSISTED_SETTINGS_SCHEMA_VERSION, SECRET_NAME_PATTERN, @@ -56,6 +57,8 @@ "SettingsUpdatePayload", "WorkspaceItem", "WorkspaceParentItem", + # Async access + "run_store", # Stores "FileSecretsStore", "FileSettingsStore", diff --git a/openhands-agent-server/openhands/agent_server/persistence/async_store.py b/openhands-agent-server/openhands/agent_server/persistence/async_store.py new file mode 100644 index 00000000000..7d2c511fe49 --- /dev/null +++ b/openhands-agent-server/openhands/agent_server/persistence/async_store.py @@ -0,0 +1,36 @@ +"""The one way async code reaches a persistence store. + +The stores (settings, secrets, workspaces, LLM/agent/meta profiles, provider +connections, MCP OAuth tokens) are synchronous: each call reads or writes +files and may wait up to its lock timeout for another writer. Called from an +``async def`` they would stall the event loop, and with it every other +request, websocket and background task, for that whole wait. + +``run_store`` runs a store call, or a sync function doing several store +calls, on a worker thread and awaits it. Keep a read-modify-write in one +function so it makes one hop:: + + settings = await run_store(store.load) + + def add(workspaces: PersistedWorkspaces) -> PersistedWorkspaces: ... + await run_store(store.update, add) + +``scripts/check_async_store_access.py`` (pre-commit) rejects store calls made +directly in async code or hopped with ``asyncio.to_thread`` / +``run_in_executor`` instead of this function. +""" + +import asyncio +from collections.abc import Callable + + +async def run_store[**P, R]( + fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs +) -> R: + """Run blocking store work ``fn(*args, **kwargs)`` off the event loop. + + Uses the loop's default executor and copies the caller's context + variables (``asyncio.to_thread`` semantics). Cancelling the awaiting task + does not stop a write already running on the worker thread. + """ + return await asyncio.to_thread(fn, *args, **kwargs) diff --git a/openhands-agent-server/openhands/agent_server/profiles_router.py b/openhands-agent-server/openhands/agent_server/profiles_router.py index 290da67fe44..be608a580c1 100644 --- a/openhands-agent-server/openhands/agent_server/profiles_router.py +++ b/openhands-agent-server/openhands/agent_server/profiles_router.py @@ -22,6 +22,7 @@ get_llm_profile_store, get_provider_connections_store, get_settings_store, + run_store, ) from openhands.sdk.llm import LLM, Message, TextContent from openhands.sdk.llm.exceptions import ( @@ -352,7 +353,9 @@ async def validate_profile( # credentials, as activation and ``switch_llm`` run it. No-op without # ``provider_connection_id``; a dangling link is a 422 naming it. with store_errors(): - llm = get_llm_profile_store().resolve_provider_connection(llm, cipher=cipher) + llm = await run_store( + get_llm_profile_store().resolve_provider_connection, llm, cipher=cipher + ) messages = [ Message( diff --git a/openhands-agent-server/openhands/agent_server/runtime_router.py b/openhands-agent-server/openhands/agent_server/runtime_router.py index fddc306f933..5e67997c4fc 100644 --- a/openhands-agent-server/openhands/agent_server/runtime_router.py +++ b/openhands-agent-server/openhands/agent_server/runtime_router.py @@ -4,6 +4,7 @@ checks are routing safeguards, not a sandbox for arbitrary shell commands. """ +import asyncio from collections.abc import Callable from copy import deepcopy from pathlib import Path @@ -38,14 +39,11 @@ async def bind_local_conversation_runtime( if event_service is None: raise HTTPException(404, "Conversation not found") request.state.runtime_event_service = event_service - workspace_root = Path( - event_service.get_conversation().workspace.working_dir - ).resolve() + workspace_dir = event_service.get_conversation().workspace.working_dir for name in ("path", "workspace_dir"): requested_path = request.path_params.get(name) or request.query_params.get(name) - if requested_path is not None and ( - not Path(requested_path).is_absolute() - or not Path(requested_path).resolve().is_relative_to(workspace_root) + if requested_path is not None and not await asyncio.to_thread( + _is_inside, Path(requested_path), Path(workspace_dir) ): raise HTTPException( 422, f"{name} must be inside the conversation workspace" @@ -62,6 +60,11 @@ async def bind_local_conversation_runtime( ) +def _is_inside(path: Path, workspace_dir: Path) -> bool: + """Whether absolute ``path`` resolves inside ``workspace_dir``.""" + return path.is_absolute() and path.resolve().is_relative_to(workspace_dir.resolve()) + + class RuntimeRouter(APIRouter): def add_api_route( self, path: str, endpoint: Callable[..., Any], **kwargs: Any diff --git a/openhands-agent-server/openhands/agent_server/settings_router.py b/openhands-agent-server/openhands/agent_server/settings_router.py index aa7a0b6245c..1cdfd6af9fa 100644 --- a/openhands-agent-server/openhands/agent_server/settings_router.py +++ b/openhands-agent-server/openhands/agent_server/settings_router.py @@ -21,6 +21,7 @@ get_llm_profile_store, get_secrets_store, get_settings_store, + run_store, ) from openhands.agent_server.persistence.models import SettingsUpdatePayload from openhands.agent_server.telemetry import notify_misc_settings_changed @@ -154,7 +155,7 @@ async def get_settings(request: Request) -> SettingsResponse: expose_mode = parse_expose_secrets_header(request) config = get_config(request) store = get_settings_store(config) - settings = store.load() or PersistedSettings() + settings = await run_store(store.load) or PersistedSettings() # Audit log all settings access for security visibility # Use WARNING level for plaintext mode to highlight security-sensitive operations @@ -455,7 +456,8 @@ async def create_mcp_server( SettingsUpdatePayload, {"agent_settings_diff": {"mcp_config": {settings_key: server_data}}}, ) - return _apply_settings_update( + return await run_store( + _apply_settings_update, request, update_data, lambda settings: _require_mcp_server_absent(settings, settings_key), @@ -478,7 +480,8 @@ async def patch_mcp_server( SettingsUpdatePayload, {"agent_settings_diff": {"mcp_config": {settings_key: patch_data}}}, ) - return _apply_settings_update( + return await run_store( + _apply_settings_update, request, update_data, lambda settings: _require_mcp_server_present(settings, settings_key), @@ -492,7 +495,8 @@ async def delete_mcp_server(request: Request, settings_key: str) -> SettingsResp SettingsUpdatePayload, {"agent_settings_diff": {"mcp_config": {settings_key: None}}}, ) - return _apply_settings_update( + return await run_store( + _apply_settings_update, request, update_data, lambda settings: _require_mcp_server_present(settings, settings_key), @@ -557,7 +561,7 @@ async def get_secret_value(request: Request, name: str) -> Response: config = get_config(request) store = get_secrets_store(config) - value = store.get_secret(name) + value = await run_store(store.get_secret, name) client_host = request.client.host if request.client else "unknown" if value is None: @@ -598,7 +602,8 @@ async def create_secret( store = get_secrets_store(config) try: - store.set_secret( + await run_store( + store.set_secret, name=secret.name, value=value, description=secret.description, @@ -644,7 +649,7 @@ async def delete_secret(request: Request, name: str) -> dict[str, bool]: client_host = request.client.host if request.client else "unknown" try: - deleted = store.delete_secret(name) + deleted = await run_store(store.delete_secret, name) except RuntimeError as e: # Data corruption protection triggered (file exists but unreadable) logger.error(f"Secret delete blocked: {e}") diff --git a/openhands-agent-server/openhands/agent_server/telemetry/service.py b/openhands-agent-server/openhands/agent_server/telemetry/service.py index a085a9c12c3..7464cb794e7 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/service.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/service.py @@ -8,11 +8,11 @@ sink. Analytics must never be able to prevent a server from starting. """ -import asyncio from functools import partial from typing import Any from openhands.agent_server.config import Config +from openhands.agent_server.persistence.async_store import run_store from openhands.agent_server.telemetry.factory import ( DiagnosticEventFactory, build_runtime_properties, @@ -49,7 +49,7 @@ def _read_misc_settings_sync(config: Config) -> dict[str, Any]: async def _read_decision(config: Config) -> TelemetryDecision: """Resolve consent from persisted settings, off the event loop.""" - misc = await asyncio.to_thread(_read_misc_settings_sync, config) + misc = await run_store(_read_misc_settings_sync, config) return resolve(misc) diff --git a/openhands-agent-server/openhands/agent_server/vscode_service.py b/openhands-agent-server/openhands/agent_server/vscode_service.py index e454655e013..2d1e3652b9a 100644 --- a/openhands-agent-server/openhands/agent_server/vscode_service.py +++ b/openhands-agent-server/openhands/agent_server/vscode_service.py @@ -210,7 +210,8 @@ async def _wait_for_exit(self) -> None: if self.process is None: return await self.process.wait() - while not await self._is_port_available(): + # Nothing signals that node released the port; polling is the only way. + while not await self._is_port_available(): # noqa: ASYNC110 await asyncio.sleep(0.1) def _proxy_script_path(self) -> Path: diff --git a/openhands-agent-server/openhands/agent_server/workspace_router.py b/openhands-agent-server/openhands/agent_server/workspace_router.py index 506627d30b3..b5dafb77dd9 100644 --- a/openhands-agent-server/openhands/agent_server/workspace_router.py +++ b/openhands-agent-server/openhands/agent_server/workspace_router.py @@ -12,6 +12,7 @@ - Path traversal outside of the workspace is rejected. """ +import asyncio from pathlib import Path from uuid import UUID @@ -54,8 +55,8 @@ async def _resolve_workspace_dir( status_code=status.HTTP_404_NOT_FOUND, detail="Conversation workspace is not local; cannot be served", ) - workspace_dir = Path(workspace.working_dir).resolve() - if not workspace_dir.is_dir(): + workspace_dir = await asyncio.to_thread(Path(workspace.working_dir).resolve) + if not await asyncio.to_thread(workspace_dir.is_dir): raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Workspace directory does not exist", diff --git a/openhands-agent-server/openhands/agent_server/workspaces_router.py b/openhands-agent-server/openhands/agent_server/workspaces_router.py index 5ee50edc587..de287b5d6cc 100644 --- a/openhands-agent-server/openhands/agent_server/workspaces_router.py +++ b/openhands-agent-server/openhands/agent_server/workspaces_router.py @@ -17,6 +17,7 @@ WorkspaceItem, WorkspaceParentItem, get_workspaces_store, + run_store, ) from openhands.sdk.logger import get_logger @@ -64,17 +65,22 @@ async def list_workspaces(request: Request) -> WorkspacesListResponse: """Return the saved workspaces and workspace parents.""" config = get_config(request) store = get_workspaces_store(config) - try: + + def read() -> tuple[PersistedWorkspaces | None, bool]: persisted = store.load() - except (OSError, PermissionError): - logger.error("Workspaces list failed - file I/O error") - raise HTTPException(status_code=500, detail="Failed to read workspaces") - if persisted is None: # ``load()`` collapses "file missing" and "file present but # unreadable/corrupted/future-schema" into the same ``None``. Returning # an empty list for the second case would hide the user's data and let # a subsequent POST overwrite the still-on-disk file with defaults. - if store._path.exists(): + return persisted, persisted is None and store._path.exists() + + try: + persisted, corrupted = await run_store(read) + except (OSError, PermissionError): + logger.error("Workspaces list failed - file I/O error") + raise HTTPException(status_code=500, detail="Failed to read workspaces") + if persisted is None: + if corrupted: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail="Workspaces file is corrupted", @@ -113,7 +119,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: ) try: - updated = store.update(apply) + updated = await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspaces add blocked: {e}") raise HTTPException( @@ -147,7 +153,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: return current.model_copy(update={"workspaces": remaining}) try: - store.update(apply) + await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspaces delete blocked: {e}") raise HTTPException( @@ -194,7 +200,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: ) try: - updated = store.update(apply) + updated = await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspace parents add blocked: {e}") raise HTTPException( @@ -228,7 +234,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: return current.model_copy(update={"workspace_parents": remaining}) try: - store.update(apply) + await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspace parents delete blocked: {e}") raise HTTPException( diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index b836a7265df..0584aa7c0dd 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -1193,7 +1193,9 @@ async def wrapped(*args: Any, **kwargs: Any) -> _T: return decorate async def _anext_with_idle_timeout( - self, iterator: AsyncIterator[_T], timeout: float + self, + iterator: AsyncIterator[_T], + timeout: float, # noqa: ASYNC109 - this function is the cancel scope ) -> _T: """Await one chunk, converting only this idle timer's own expiry. diff --git a/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py b/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py index 796d629da1a..bb25709870a 100644 --- a/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py +++ b/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py @@ -78,7 +78,7 @@ async def start_command( self, command: str, cwd: str | Path | None = None, - timeout: float = 30, + timeout: float = 30, # noqa: ASYNC109 - sent to the server's command ) -> str: """Start a command and return its ID without waiting for completion.""" return await self._execute(self._start_command_generator(command, cwd, timeout)) @@ -109,7 +109,7 @@ async def execute_command( self, command: str, cwd: str | Path | None = None, - timeout: float = 30.0, + timeout: float = 30.0, # noqa: ASYNC109 - sent to the server's command ) -> CommandResult: """Execute a bash command on the remote system. diff --git a/pyproject.toml b/pyproject.toml index 32202cc20c0..bde8e492b67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ select = [ "I", # isort "UP", # pyupgrade "ARG", # flake8-unused-arguments + "ASYNC", # flake8-async: no blocking calls on the event loop ] # Enforce rules that catch mutable defaults and related pitfalls # - B006: mutable-argument-default @@ -78,7 +79,8 @@ ignore = ["UP042"] [tool.ruff.lint.per-file-ignores] # Test files often have unused arguments (fixtures, mocks, interface implementations) -"tests/**/*.py" = ["ARG"] +# and poll or block on purpose while driving threads and subprocesses. +"tests/**/*.py" = ["ARG", "ASYNC"] # Allowlist safe default calls for flake8-bugbear rules (e.g., FastAPI Depends) diff --git a/scripts/check_async_store_access.py b/scripts/check_async_store_access.py new file mode 100644 index 00000000000..b35d0942e3b --- /dev/null +++ b/scripts/check_async_store_access.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Keep blocking store I/O off the agent-server event loop. + +The persistence stores (settings, secrets, workspaces, LLM/agent/meta +profiles, provider connections, MCP OAuth tokens, runtime provisioning) are +synchronous: they read and write files under a file lock. Inside an ``async +def`` they must be reached through ``run_store`` from +``openhands.agent_server.persistence.async_store``, the one place where store +I/O hops to a worker thread. + +This check flags, inside every ``async def`` under ``openhands-agent-server``: + +1. a direct method call on a store (``store.load()``, + ``get_settings_store(config).update(fn)``, ``registry.provisioning.save(x)``); +2. store work handed to any other thread hop (``asyncio.to_thread``, + ``loop.run_in_executor``, ``run_in_threadpool``) instead of ``run_store``: + a store method as the hopped callable, or a lambda that calls one. + +A store expression is: + +- a local name annotated with a store type or assigned from a store getter or + constructor, or a getter call used inline; +- an attribute (``self.x`` or ``obj.x``) whose name some agent-server class + declares with a store type or assigns from a store getter or constructor. + Attribute names are matched across classes, so give a store attribute a + name no non-store attribute uses. + +Code in nested sync ``def``s and lambdas is not checked: those run wherever +their caller puts them, normally inside ``run_store``. + +Limits: a store held in a container or reached through an unannotated +parameter is not seen, and a sync helper that does store I/O and is called +directly from async code is not followed. Methods in ``_PURE_METHODS`` touch +no file and are allowed. + +Usage: check_async_store_access.py [FILES...] +(no files: scan openhands-agent-server) +""" + +import ast +import sys +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCAN_ROOT = ROOT / "openhands-agent-server" / "openhands" / "agent_server" + +STORE_TYPES = frozenset( + { + "SettingsStore", + "FileSettingsStore", + "SecretsStore", + "FileSecretsStore", + "WorkspacesStore", + "FileWorkspacesStore", + "LLMProfileStore", + "AgentProfileStore", + "MetaProfileStore", + "ProviderConnectionStore", + "MCPSettingsOAuthTokenStore", + "RuntimeProvisioningStore", + } +) + +STORE_GETTERS = frozenset( + { + "get_settings_store", + "get_secrets_store", + "get_workspaces_store", + "get_provider_connections_store", + "get_llm_profile_store", + "get_agent_profile_store", + } +) + +# Methods that only build or inspect an object and touch no file. +_PURE_METHODS = frozenset({"with_cipher", "manifest_path"}) + +THREAD_HOPS = frozenset({"to_thread", "run_in_executor", "run_in_threadpool"}) +FACADE = "run_store" + +_NESTED_SCOPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef) + + +@dataclass(frozen=True) +class Violation: + path: Path + line: int + message: str + + def __str__(self) -> str: + try: + shown = self.path.relative_to(ROOT) + except ValueError: + shown = self.path + return f"{shown}:{self.line}: {self.message}" + + +def _annotation_names(node: ast.expr | None) -> Iterator[str]: + """Every bare or dotted type name mentioned in an annotation.""" + if node is None: + return + for sub in ast.walk(node): + if isinstance(sub, ast.Name): + yield sub.id + elif isinstance(sub, ast.Attribute): + yield sub.attr + elif isinstance(sub, ast.Constant) and isinstance(sub.value, str): + try: + parsed = ast.parse(sub.value, mode="eval").body + except SyntaxError: + continue + yield from _annotation_names(parsed) + + +def _is_store_annotation(node: ast.expr | None) -> bool: + return any(name in STORE_TYPES for name in _annotation_names(node)) + + +def _callee_name(call: ast.Call) -> str | None: + if isinstance(call.func, ast.Name): + return call.func.id + if isinstance(call.func, ast.Attribute): + return call.func.attr + return None + + +def _makes_store(value: ast.expr | None) -> bool: + """True for ``get_x_store(...)`` or ``XStore(...)`` (possibly awaited).""" + if isinstance(value, ast.Await): + value = value.value + if not isinstance(value, ast.Call): + return False + name = _callee_name(value) + return name in STORE_GETTERS or name in STORE_TYPES + + +def store_attributes(tree: ast.AST) -> set[str]: + """Attribute names that classes in ``tree`` use to hold a store.""" + attrs: set[str] = set() + for cls in ast.walk(tree): + if not isinstance(cls, ast.ClassDef): + continue + for node in ast.walk(cls): + if isinstance(node, ast.AnnAssign): + target = node.target + if isinstance(target, ast.Name | ast.Attribute) and ( + _is_store_annotation(node.annotation) or _makes_store(node.value) + ): + attrs.add( + target.id if isinstance(target, ast.Name) else target.attr + ) + elif isinstance(node, ast.Assign) and _makes_store(node.value): + attrs.update( + target.attr + for target in node.targets + if isinstance(target, ast.Attribute) + ) + return attrs + + +def _own_nodes(func: ast.AsyncFunctionDef) -> Iterator[ast.AST]: + """Nodes of ``func``'s body, not descending into nested scopes.""" + stack: list[ast.AST] = [ + stmt for stmt in func.body if not isinstance(stmt, _NESTED_SCOPES) + ] + while stack: + node = stack.pop() + yield node + stack.extend( + child + for child in ast.iter_child_nodes(node) + if not isinstance(child, _NESTED_SCOPES) + ) + + +class _AsyncFunctionChecker: + def __init__( + self, path: Path, func: ast.AsyncFunctionDef, store_attrs: set[str] + ) -> None: + self.path = path + self.func = func + self.store_attrs = store_attrs + self.store_names = self._collect_store_names() + + def _collect_store_names(self) -> set[str]: + names: set[str] = set() + args = self.func.args + for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs]: + if _is_store_annotation(arg.annotation): + names.add(arg.arg) + for node in _own_nodes(self.func): + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if _is_store_annotation(node.annotation) or _makes_store(node.value): + names.add(node.target.id) + elif isinstance(node, ast.Assign) and _makes_store(node.value): + for target in node.targets: + if isinstance(target, ast.Name): + names.add(target.id) + return names + + def _is_store(self, node: ast.expr) -> bool: + if isinstance(node, ast.Name): + return node.id in self.store_names + if isinstance(node, ast.Attribute): + return node.attr in self.store_attrs + return _makes_store(node) + + def _store_method(self, node: ast.expr) -> str | None: + """``m`` when ``node`` is ``.m`` for a method that does I/O.""" + if ( + isinstance(node, ast.Attribute) + and node.attr not in _PURE_METHODS + and self._is_store(node.value) + ): + return node.attr + return None + + def _hopped_store_work(self, call: ast.Call, callee: str) -> str | None: + """The store method a thread hop runs, if its callable is store work. + + Only the callable position counts (``run_in_executor`` takes the + executor first); a store method passed further along is a callback + for the hopped function, not the hop's own work. + """ + position = 1 if callee == "run_in_executor" else 0 + if len(call.args) <= position: + return None + hopped = call.args[position] + if isinstance(hopped, ast.Lambda): + for sub in ast.walk(hopped.body): + if isinstance(sub, ast.Call): + method = self._store_method(sub.func) + if method is not None: + return method + return None + return self._store_method(hopped) + + def violations(self) -> Iterator[Violation]: + for node in _own_nodes(self.func): + if not isinstance(node, ast.Call): + continue + method = self._store_method(node.func) + if method is not None: + yield Violation( + self.path, + node.lineno, + f"blocking store call .{method}() in async " + f"{self.func.name}(); await {FACADE}(...) instead", + ) + continue + callee = _callee_name(node) + if callee not in THREAD_HOPS: + continue + hopped = self._hopped_store_work(node, callee) + if hopped is not None: + yield Violation( + self.path, + node.lineno, + f"store work .{hopped} handed to {callee}() in " + f"async {self.func.name}(); use {FACADE}(...)", + ) + + +def check_source( + path: Path, source: str, store_attrs: Iterable[str] = () +) -> list[Violation]: + """Violations in one module; ``store_attrs`` adds attribute names declared + elsewhere (the module's own classes are always read).""" + tree = ast.parse(source, filename=str(path)) + attrs = store_attributes(tree) | set(store_attrs) + found = [ + violation + for node in ast.walk(tree) + if isinstance(node, ast.AsyncFunctionDef) + for violation in _AsyncFunctionChecker(path, node, attrs).violations() + ] + return sorted(found, key=lambda v: v.line) + + +def _targets(argv: list[str]) -> Iterable[Path]: + if not argv: + return sorted(SCAN_ROOT.rglob("*.py")) + paths = [Path(arg).resolve() for arg in argv] + return [p for p in paths if p.suffix == ".py" and p.is_relative_to(SCAN_ROOT)] + + +def main(argv: list[str]) -> int: + # Store attributes are collected from the whole package even when only + # some files are checked: ``registry.provisioning`` is declared elsewhere. + sources = { + path: path.read_text(encoding="utf-8") for path in SCAN_ROOT.rglob("*.py") + } + store_attrs: set[str] = set() + for source in sources.values(): + store_attrs |= store_attributes(ast.parse(source)) + violations: list[Violation] = [] + for path in _targets(argv): + source = sources.get(path) or path.read_text(encoding="utf-8") + violations.extend(check_source(path, source, store_attrs)) + for violation in violations: + print(violation) + if violations: + print( + f"\n{len(violations)} blocking store access(es) in async code. " + f"Wrap the store work in a sync function and await {FACADE}(fn, ...)." + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/agent_server/stress/test_slow_webhook.py b/tests/agent_server/stress/test_slow_webhook.py index 4de818d960e..ff0ea5f6416 100644 --- a/tests/agent_server/stress/test_slow_webhook.py +++ b/tests/agent_server/stress/test_slow_webhook.py @@ -185,7 +185,7 @@ async def test_slow_webhook_does_not_unbound_growth( (b) the webhook subscriber buffer doesn't accumulate unbounded events. """ # Distinct workspaces per run so any workspace-side state from the - # baseline (e.g. .git from `_ensure_workspace_is_git_repo`, scratch + # baseline (e.g. .git from `_prepare_workspace_dir`, scratch # files) doesn't bleed into the webhook timing. workspace_baseline = str(tmp_path / "ws_baseline") workspace_webhook = str(tmp_path / "ws_webhook") diff --git a/tests/agent_server/test_profile_store_lock_responsiveness.py b/tests/agent_server/test_profile_store_lock_responsiveness.py index 94384039a30..5fa5c719ff2 100644 --- a/tests/agent_server/test_profile_store_lock_responsiveness.py +++ b/tests/agent_server/test_profile_store_lock_responsiveness.py @@ -72,6 +72,30 @@ 200, id="provider-connections", ), + pytest.param( + ".workspaces.lock", + "POST", + "/api/workspaces", + {"workspaces": [{"id": "/qa-busy", "name": "qa", "path": "/qa-busy"}]}, + 200, + id="workspaces", + ), + pytest.param( + ".secrets.lock", + "PUT", + "/api/settings/secrets", + {"name": "QA_BUSY", "value": "qa-value"}, + 200, + id="secrets", + ), + pytest.param( + ".settings.lock", + "POST", + "/api/settings/mcp/qa-busy", + {"transport": "http", "url": "https://qa-busy.example/mcp"}, + 201, + id="settings-mcp-server", + ), ], ) async def test_alive_answers_while_store_request_waits_for_lock( diff --git a/tests/cross/test_check_async_store_access.py b/tests/cross/test_check_async_store_access.py new file mode 100644 index 00000000000..3a73667a2ce --- /dev/null +++ b/tests/cross/test_check_async_store_access.py @@ -0,0 +1,143 @@ +"""Tests for the pre-commit check that keeps store I/O off the event loop.""" + +import importlib.util +import textwrap +from pathlib import Path +from types import ModuleType + +import pytest + + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "check_async_store_access.py" + + +@pytest.fixture(scope="module") +def checker() -> ModuleType: + spec = importlib.util.spec_from_file_location("check_async_store_access", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _lines(checker: ModuleType, source: str) -> list[int]: + violations = checker.check_source(Path("mod.py"), textwrap.dedent(source)) + return [violation.line for violation in violations] + + +@pytest.mark.parametrize( + "source", + [ + pytest.param( + """ + async def route(config): + store = get_settings_store(config) + return store.load() + """, + id="local-from-getter", + ), + pytest.param( + """ + async def route(config): + return get_secrets_store(config).get_secret("X") + """, + id="inline-getter", + ), + pytest.param( + """ + async def route(store: FileWorkspacesStore | None): + store.update(lambda w: w) + """, + id="annotated-parameter", + ), + pytest.param( + """ + class Registry: + provisioning: RuntimeProvisioningStore + + async def route(registry): + return registry.provisioning.load(1) + """, + id="attribute-through-other-object", + ), + pytest.param( + """ + async def route(store: LLMProfileStore): + return await asyncio.to_thread(store.list_summaries) + """, + id="to-thread-hop", + ), + pytest.param( + """ + async def route(config): + return await asyncio.to_thread( + lambda: get_settings_store(config).load() or None + ) + """, + id="lambda-hop", + ), + pytest.param( + """ + async def route(loop, store: FileSecretsStore): + return await loop.run_in_executor(None, store.get_secret, "X") + """, + id="run-in-executor-hop", + ), + ], +) +def test_flags_store_io_on_the_event_loop(checker: ModuleType, source: str) -> None: + assert len(_lines(checker, source)) == 1 + + +@pytest.mark.parametrize( + "source", + [ + pytest.param( + """ + async def route(config): + store = get_settings_store(config) + return await run_store(store.load) + """, + id="facade", + ), + pytest.param( + """ + async def route(store: FileWorkspacesStore): + def read(): + return store.load(), store.load() + return await run_store(read) + """, + id="nested-sync-helper", + ), + pytest.param( + """ + def route(config): + return get_settings_store(config).load() + """, + id="sync-route", + ), + pytest.param( + """ + async def route(registry: Registry, store: LLMProfileStore): + path = registry.provisioning.manifest_path(1) + return store.with_cipher(None), path + """, + id="pure-methods", + ), + pytest.param( + """ + async def route(loop, store: FileSecretsStore): + return await loop.run_in_executor(None, probe, store.get_secret) + """, + id="store-method-as-callback", + ), + ], +) +def test_allows_store_io_off_the_event_loop(checker: ModuleType, source: str) -> None: + assert _lines(checker, source) == [] + + +def test_agent_server_has_no_blocking_store_access( + checker: ModuleType, capsys: pytest.CaptureFixture[str] +) -> None: + assert checker.main([]) == 0, capsys.readouterr().out