diff --git a/.agents/skills/verify-sdk/programs/S21/acp_conversation.py b/.agents/skills/verify-sdk/programs/S21/acp_conversation.py index 403610d5221..2cd6bf97235 100644 --- a/.agents/skills/verify-sdk/programs/S21/acp_conversation.py +++ b/.agents/skills/verify-sdk/programs/S21/acp_conversation.py @@ -562,7 +562,8 @@ async def cancel_mid_prompt() -> str: conversation.send_message("[cancel] build everything") task = asyncio.create_task(conversation.arun()) deadline = time.monotonic() + 30 - while not journal("cancel_waiting") and time.monotonic() < deadline: + # The fake ACP server only writes a journal file; nothing to await. + while not journal("cancel_waiting") and time.monotonic() < deadline: # noqa: ASYNC110 await asyncio.sleep(0.1) await asyncio.sleep(0.3) task.cancel() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 08eb2cbc5ba..c48b88be15e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,6 +69,14 @@ repos: types: [python] pass_filenames: true always_run: false + - id: check-async-store-access + name: Keep blocking store I/O off the agent-server event loop + entry: uv + args: [run, python, scripts/check_async_store_access.py] + language: system + files: ^openhands-agent-server/.*\.py$ + pass_filenames: true + always_run: false - id: check-secret-fields name: Keep secrets on SecretField entry: uv diff --git a/openhands-agent-server/openhands/agent_server/agent_profiles_router.py b/openhands-agent-server/openhands/agent_server/agent_profiles_router.py index 49227268d57..707aeeea048 100644 --- a/openhands-agent-server/openhands/agent_server/agent_profiles_router.py +++ b/openhands-agent-server/openhands/agent_server/agent_profiles_router.py @@ -35,6 +35,7 @@ FileSettingsStore, PersistedSettings, ServerStores, + run_store, ) from openhands.agent_server.profiles_router import MAX_PROFILES, _has_api_key from openhands.sdk.errors import Conflict, NotFound, OperationFailed, Unprocessable @@ -582,11 +583,11 @@ async def materialize_agent_profile( store = stores.agent_profiles try: # Off the event loop: the store lock wait blocks (up to 30 s). - profile = await asyncio.to_thread(store.load, name) + profile = await run_store(store.load, name) except FileNotFoundError: raise NotFound(f"Agent profile '{name}' not found") - settings = stores.settings.load() or PersistedSettings() + settings = await run_store(stores.settings.load) or PersistedSettings() context = settings.agent_settings.agent_context return await asyncio.to_thread( lambda: preview_launch( diff --git a/openhands-agent-server/openhands/agent_server/api.py b/openhands-agent-server/openhands/agent_server/api.py index d919cb7191c..49d1068fbfc 100644 --- a/openhands-agent-server/openhands/agent_server/api.py +++ b/openhands-agent-server/openhands/agent_server/api.py @@ -78,7 +78,7 @@ check_openai_api_key, openai_router, ) -from openhands.agent_server.persistence import reencrypt_stale_stores +from openhands.agent_server.persistence import reencrypt_stale_stores, run_store from openhands.agent_server.plugins_router import plugins_router from openhands.agent_server.profiles_router import profiles_router from openhands.agent_server.provider_connections_router import ( @@ -419,7 +419,7 @@ async def stop_tool_preload_service(): # Non-deferred (legacy) path: build and enter the conversation # service as part of the lifespan, exactly as before. - await asyncio.to_thread(reencrypt_stale_stores, context.stores) + await run_store(reencrypt_stale_stores, context.stores) # # A key that came from the config file is visible only in ``config``; # in-process callers of this server read it from the environment. diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py index 7c7019abc91..773f29d87f8 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/backend.py @@ -456,7 +456,7 @@ async def prepare(self, name: str, revision: str) -> BackendStatus: if ( prepared is not None and prepared.revision == revision - and Path(prepared.artifact_dir).is_dir() + and await asyncio.to_thread(Path(prepared.artifact_dir).is_dir) ): return self._status_unlocked(name) @@ -790,19 +790,20 @@ async def start(self, name: str, revision: str) -> BackendStatus: if prepared is None or prepared.revision != revision: raise RuntimeError("backend revision must be prepared before start") artifact_dir = Path(prepared.artifact_dir) - if not artifact_dir.is_dir(): + if not await asyncio.to_thread(artifact_dir.is_dir): raise RuntimeError("prepared backend artifact is missing") parts = self._backend_parts(name) assert parts is not None backend, artifact, current_platform = parts - expected_artifact_dir = ( - self.artifacts_dir / name / prepared.artifact_sha256 - ).resolve() + expected_artifact_dir = await asyncio.to_thread( + (self.artifacts_dir / name / prepared.artifact_sha256).resolve + ) if ( prepared.platform != current_platform or prepared.artifact_sha256 != artifact.sha256 - or artifact_dir.resolve() != expected_artifact_dir + or await asyncio.to_thread(artifact_dir.resolve) + != expected_artifact_dir ): raise RuntimeError( "prepared backend metadata does not match manifest" @@ -918,8 +919,12 @@ def _runtime_group_alive(self, runtime: _Runtime) -> bool: process.returncode is None or self._group_alive(process.pid) ) - async def _wait_for_group_exit(self, pgid: int, timeout: float) -> bool: - deadline = time.monotonic() + timeout + async def _wait_for_group_exit(self, pgid: int) -> bool: + """Whether the group exits within ``_STOP_TIMEOUT_SECONDS``. + + Nothing signals a process group's exit, so this polls. + """ + deadline = time.monotonic() + _STOP_TIMEOUT_SECONDS while True: if not self._group_alive(pgid): return True @@ -938,14 +943,14 @@ async def _terminate_runtime(self, runtime: _Runtime) -> None: # on the leader's exit code would leak those descendants. if process.returncode is None: self._signal_group(pgid, signal.SIGTERM) - if not await self._wait_for_group_exit(pgid, _STOP_TIMEOUT_SECONDS): + if not await self._wait_for_group_exit(pgid): self._signal_group(pgid, signal.SIGKILL) - await self._wait_for_group_exit(pgid, _STOP_TIMEOUT_SECONDS) + await self._wait_for_group_exit(pgid) elif self._group_alive(pgid): # With no leader left to shut down cleanly, its descendants are # orphaned; do not let one hold status or restart for the grace period. self._signal_group(pgid, signal.SIGKILL) - await self._wait_for_group_exit(pgid, _STOP_TIMEOUT_SECONDS) + await self._wait_for_group_exit(pgid) if process.returncode is None: await process.wait() await self._finish_runtime(runtime) @@ -1004,11 +1009,9 @@ async def reap_orphans(self) -> None: continue # Its server is still running and owns it. if self._is_recorded_backend(record): self._signal_group(record.pgid, signal.SIGTERM) - if not await self._wait_for_group_exit( - record.pgid, _STOP_TIMEOUT_SECONDS - ): + if not await self._wait_for_group_exit(record.pgid): self._signal_group(record.pgid, signal.SIGKILL) - await self._wait_for_group_exit(record.pgid, _STOP_TIMEOUT_SECONDS) + await self._wait_for_group_exit(record.pgid) path.unlink(missing_ok=True) def has_running_backends(self) -> bool: diff --git a/openhands-agent-server/openhands/agent_server/conversation_router.py b/openhands-agent-server/openhands/agent_server/conversation_router.py index ac5f3a255ad..a929110d6c9 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_router.py +++ b/openhands-agent-server/openhands/agent_server/conversation_router.py @@ -55,6 +55,7 @@ UpdateSecretsRequest, trim_conversation_response_skills, ) +from openhands.agent_server.persistence import run_store from openhands.sdk import LLM from openhands.sdk.conversation.state import ConversationExecutionStatus from openhands.sdk.conversation.types import ( @@ -645,7 +646,9 @@ async def switch_conversation_llm( # reference or missing credential surfaces as 422 before the working LLM # is replaced. Inline-key configs without a provider_connection_id are # byte-identical to the old path. - llm = llm_profile_store.resolve_provider_connection(llm, cipher=cipher) + llm = await run_store( + llm_profile_store.resolve_provider_connection, llm, cipher=cipher + ) try: conversation.switch_llm(llm) except ValueError as e: diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 7fdcda7b66f..98ee42fb71d 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -54,7 +54,11 @@ StoredConversation, UpdateConversationRequest, ) -from openhands.agent_server.persistence import PersistedSettings, ServerStores +from openhands.agent_server.persistence import ( + PersistedSettings, + ServerStores, + run_store, +) from openhands.agent_server.pub_sub import Subscriber from openhands.agent_server.server_details_router import update_last_execution_time from openhands.agent_server.telemetry import ( @@ -812,10 +816,7 @@ def _is_codex_agent(agent: AgentBase | None) -> bool: async def _has_local_codex_credential(self) -> bool: if self.stores is None: return False - value = await asyncio.to_thread( - self.stores.secrets.get_secret, - CODEX_AUTH_SECRET_NAME, - ) + value = await run_store(self.stores.secrets.get_secret, CODEX_AUTH_SECRET_NAME) return value is not None async def _load_settings(self) -> PersistedSettings: @@ -827,7 +828,7 @@ async def _load_settings(self) -> PersistedSettings: # every launch reads it, so a bad file mode must not take down request # shapes that need nothing from settings. try: - settings = await asyncio.to_thread(stores.settings.load) + settings = await run_store(stores.settings.load) except (PermissionError, OSError): logger.warning( "Cannot read settings; starting without the stored agent preferences", @@ -2822,7 +2823,7 @@ async def __call__(self, event: Event) -> None: # truncation. This keeps auto-titling non-breaking for consumers who # don't configure title_llm_profile. conversation = self.service._conversation - title_llm = self._load_title_llm() + title_llm = await run_store(self._load_title_llm) if title_llm is None: title_llm = conversation.agent.llm if conversation else None diff --git a/openhands-agent-server/openhands/agent_server/credential_binding.py b/openhands-agent-server/openhands/agent_server/credential_binding.py index 7e7e17f261a..abcd189f850 100644 --- a/openhands-agent-server/openhands/agent_server/credential_binding.py +++ b/openhands-agent-server/openhands/agent_server/credential_binding.py @@ -11,7 +11,7 @@ from openhands.agent_server.conversation_service import ConversationService from openhands.agent_server.dependencies import get_conversation_service from openhands.agent_server.event_service import CredentialBindingActivationTooLate -from openhands.agent_server.persistence import FileSecretsStore +from openhands.agent_server.persistence import FileSecretsStore, run_store from openhands.sdk.agent.acp_file_credentials import supports_file_credential_binding from openhands.sdk.credential import ( CredentialAuthorizationRejected, @@ -58,7 +58,7 @@ class LocalVersionedCredentialBinding: async def load(self) -> ResolvedCredential: try: - value, version = await asyncio.to_thread( + value, version = await run_store( self.store.load_versioned_secret, self.secret_name, ) @@ -72,7 +72,7 @@ async def load(self) -> ResolvedCredential: async def replace(self, expected_version: str, value: str) -> str: try: - return await asyncio.to_thread( + return await run_store( self.store.replace_versioned_secret, self.secret_name, expected_version, diff --git a/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py b/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py index c735f266633..75659665145 100644 --- a/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py +++ b/openhands-agent-server/openhands/agent_server/docker_runtime/proxy.py @@ -94,7 +94,7 @@ async def proxy_http( workspace: ProxyTarget, *, upstream_path: str, - timeout: float | None = None, + read_timeout: float | None = None, body: bytes | None = None, on_close: Callable[[], Any] | None = None, reject_redirects: bool = False, @@ -108,8 +108,8 @@ async def proxy_http( agent-server to forward to. Typically the same path the outer server received, since the inner agent-server exposes the same API surface. - timeout: Per-request timeout in seconds. ``None`` (the default) means - no read timeout — conversation event streams can be long-lived. + read_timeout: Upstream read timeout in seconds. ``None`` (the default) + means no read timeout — conversation event streams can be long-lived. body: Replacement request body. By default the incoming body is streamed. on_close: Awaited or invoked once the streamed response is fully consumed, client disconnects, or connection fails. Callers use this to release @@ -157,7 +157,9 @@ async def _request_body() -> AsyncIterator[bytes]: try: client = await stack.enter_async_context( httpx.AsyncClient( - timeout=httpx.Timeout(connect=10.0, read=timeout, write=30.0, pool=10.0) + timeout=httpx.Timeout( + connect=10.0, read=read_timeout, write=30.0, pool=10.0 + ) ) ) upstream = await stack.enter_async_context( diff --git a/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py b/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py index 3d98e2ccb5d..7e34b2c0de0 100644 --- a/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py +++ b/openhands-agent-server/openhands/agent_server/docker_runtime/routers.py @@ -47,7 +47,7 @@ StartConversationRequest, UpdateSecretsRequest, ) -from openhands.agent_server.persistence import PersistedSettings +from openhands.agent_server.persistence import PersistedSettings, run_store from openhands.agent_server.utils import safe_rmtree from openhands.sdk.conversation.request import ConversationContainerOptions from openhands.sdk.errors import ( @@ -147,6 +147,32 @@ async def detach() -> None: ) +def _worktree_git_paths(workspace_path: Path) -> dict[str, Path]: + """The repository and git common dir a worktree start records, if any.""" + try: + validate_git_repository(workspace_path) + repo_root = Path( + run_git_command( + ["git", "--no-pager", "rev-parse", "--show-toplevel"], + workspace_path, + ) + ).resolve() + git_common_dir = Path( + run_git_command( + ["git", "--no-pager", "rev-parse", "--git-common-dir"], + workspace_path, + ) + ) + except (GitCommandError, GitRepositoryError): + return {} + if not git_common_dir.is_absolute(): + git_common_dir = workspace_path / git_common_dir + return { + "worktree_repository_path": repo_root, + "worktree_git_common_dir_path": git_common_dir.resolve(), + } + + @docker_conversation_router.post("") async def start_conversation( request: Request, @@ -212,33 +238,14 @@ async def start_docker_conversation( payload, launched, options, worktree = await _prepare_forward( body, registry, existing=existing ) - identity = registry.provisioning.create(conversation_id, host_workspace) + identity = await run_store( + registry.provisioning.create, conversation_id, host_workspace + ) updates: dict[str, Any] = {} if worktree: - try: - validate_git_repository(identity.workspace_path) - repo_root = Path( - run_git_command( - ["git", "--no-pager", "rev-parse", "--show-toplevel"], - identity.workspace_path, - ) - ).resolve() - git_common_dir = Path( - run_git_command( - ["git", "--no-pager", "rev-parse", "--git-common-dir"], - identity.workspace_path, - ) - ) - if not git_common_dir.is_absolute(): - git_common_dir = identity.workspace_path / git_common_dir - git_common_dir = git_common_dir.resolve() - except (GitCommandError, GitRepositoryError): - pass - else: - updates.update( - worktree_repository_path=repo_root, - worktree_git_common_dir_path=git_common_dir, - ) + updates.update( + await asyncio.to_thread(_worktree_git_paths, identity.workspace_path) + ) if launched is not None and identity.launched_agent_profile is None: updates["launched_agent_profile"] = launched # Fixed once the container accepted the conversation; until then a @@ -247,7 +254,7 @@ async def start_docker_conversation( updates["container"] = options if updates: identity = identity.model_copy(update=updates) - registry.provisioning.save(identity) + await run_store(registry.provisioning.save, identity) container = await registry.get_or_create(conversation_id) release = registry.acquire_lease(conversation_id) forwarded = await asyncio.to_thread(payload, identity.cipher) @@ -310,7 +317,7 @@ async def _prepare_forward( ), ) try: - settings = await asyncio.to_thread(registry.stores.settings.load) + settings = await run_store(registry.stores.settings.load) except (OSError, PermissionError): settings = None settings = settings or PersistedSettings() @@ -359,7 +366,7 @@ async def runtime_credentials( registry = get_registry(conversation_registry) if not registry.conversation_dir(conversation_id).joinpath("meta.json").is_file(): raise NotFound("Conversation not found") - identity = registry.provisioning.load(conversation_id) + identity = await run_store(registry.provisioning.load, conversation_id) return JSONResponse({"session_api_key": identity.api_key.get_secret_value()}) @@ -421,9 +428,10 @@ async def delete_conversation( logger.exception( "Could not clean up conversation worktree %s", conversation_id ) - await asyncio.to_thread( - safe_rmtree, registry.provisioning.runtime_dir(conversation_id) + runtime_dir = await run_store( + registry.provisioning.runtime_dir, conversation_id ) + await asyncio.to_thread(safe_rmtree, runtime_dir) await asyncio.to_thread(safe_rmtree, registry.conversation_dir(conversation_id)) registry.provisioning.manifest_path(conversation_id).unlink(missing_ok=True) registry.archived_marker(conversation_id).unlink(missing_ok=True) @@ -481,7 +489,8 @@ async def proxy_conversation( update = UpdateSecretsRequest.model_validate_json(await request.body()) except ValueError as exc: raise Unprocessable("Invalid secrets payload") from exc - profile = registry.provisioning.load(conversation_id).launched_agent_profile + identity = await run_store(registry.provisioning.load, conversation_id) + profile = identity.launched_agent_profile secrets = update.secrets if profile is not None: secrets = { diff --git a/openhands-agent-server/openhands/agent_server/event_router.py b/openhands-agent-server/openhands/agent_server/event_router.py index 0fc95a26003..10fdd13eeb6 100644 --- a/openhands-agent-server/openhands/agent_server/event_router.py +++ b/openhands-agent-server/openhands/agent_server/event_router.py @@ -42,6 +42,7 @@ FileSettingsStore, PersistedSettings, SettingsStore, + run_store, ) from openhands.sdk import Message from openhands.sdk.conversation.state import ConversationExecutionStatus @@ -399,9 +400,7 @@ async def respond_to_confirmation( async def require_reactions_enabled(settings_store: SettingsStore) -> None: """Refuse with 409 unless the user has conversation reactions on.""" try: - settings = await asyncio.to_thread( - lambda: settings_store.load() or PersistedSettings() - ) + settings = await run_store(settings_store.load) or PersistedSettings() except (PermissionError, OSError): raise Conflict("Cannot read the settings file to check the feature flag") if not is_enabled(settings.experimental_features, "conversation_reactions"): diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index cd737a3056e..90a3d39d550 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1239,8 +1239,8 @@ def stats_callback() -> None: conversation.llm_registry.set_stats_update_callback(stats_callback) @staticmethod - def _ensure_workspace_is_git_repo(working_dir: Path) -> None: - """Initialize the workspace as a git repo if it isn't already one. + def _prepare_workspace_dir(working_dir: Path) -> None: + """Create the workspace and make it a git repo if it isn't already one. The /api/git/changes endpoint expects a real repository to compute changes against; without this, agent-created files never appear in @@ -1248,6 +1248,7 @@ def _ensure_workspace_is_git_repo(working_dir: Path) -> None: are handled by `get_valid_ref()` via GIT_EMPTY_TREE_HASH, and untracked files surface through `git ls-files --others`. """ + working_dir.mkdir(parents=True, exist_ok=True) try: validate_git_repository(working_dir) return # already a repo @@ -1285,9 +1286,9 @@ async def start(self): await self._scrub_persisted_credentials() workspace = self.stored.workspace assert isinstance(workspace, LocalWorkspace) - working_dir = Path(workspace.working_dir) - working_dir.mkdir(parents=True, exist_ok=True) - self._ensure_workspace_is_git_repo(working_dir) + await asyncio.to_thread( + self._prepare_workspace_dir, Path(workspace.working_dir) + ) # base_state.json is the single source of truth for the agent. On resume # (base_state exists) pass ``agent=None`` so LocalConversation keeps the # persisted agent. On a new conversation the creating caller supplied the @@ -1805,8 +1806,11 @@ async def _run_and_publish(): # coroutine enters its own try/finally. self._run_task.add_done_callback(settle) + # ``timeout`` bounds only the wait for the run task, not the status read + # after it, which a caller-side ``asyncio.timeout`` would also cut off. async def wait_for_run_completion( - self, timeout: float | None = None + self, + timeout: float | None = None, # noqa: ASYNC109 ) -> ConversationExecutionStatus: """Wait for the active run task without cancelling it on timeout.""" run_task = self._run_task diff --git a/openhands-agent-server/openhands/agent_server/experimental_features_router.py b/openhands-agent-server/openhands/agent_server/experimental_features_router.py index 537086d5222..5fc7d67d9da 100644 --- a/openhands-agent-server/openhands/agent_server/experimental_features_router.py +++ b/openhands-agent-server/openhands/agent_server/experimental_features_router.py @@ -1,7 +1,5 @@ """Routes for user-facing experimental features (see :mod:`.experimental_features`).""" -import asyncio - from fastapi import APIRouter, Depends from openhands.agent_server.dependencies import get_settings_store @@ -13,7 +11,11 @@ find_experimental_feature, is_enabled, ) -from openhands.agent_server.persistence import FileSettingsStore, PersistedSettings +from openhands.agent_server.persistence import ( + FileSettingsStore, + PersistedSettings, + run_store, +) from openhands.sdk.errors import Conflict, NotFound @@ -27,7 +29,7 @@ async def list_experimental_features( store: FileSettingsStore = Depends(get_settings_store), ) -> list[ExperimentalFeature]: """List the features the user can turn on or off, and whether each is on.""" - settings = await asyncio.to_thread(lambda: store.load() or PersistedSettings()) + settings = await run_store(store.load) or PersistedSettings() return experimental_feature_flags(settings.experimental_features) diff --git a/openhands-agent-server/openhands/agent_server/file_router.py b/openhands-agent-server/openhands/agent_server/file_router.py index 8985003e3ba..c963a579d36 100644 --- a/openhands-agent-server/openhands/agent_server/file_router.py +++ b/openhands-agent-server/openhands/agent_server/file_router.py @@ -102,18 +102,20 @@ async def _upload_file(path: str, file: UploadFile) -> FileUploadResult: if not target_path.is_absolute(): raise Invalid("Path must be absolute") - # Ensure target directory exists - target_path.parent.mkdir(parents=True, exist_ok=True) + await asyncio.to_thread(target_path.parent.mkdir, parents=True, exist_ok=True) # Stream the file to disk to avoid memory issues with large files. - # Offload writes to a worker thread so slow storage (NFS, FUSE, - # encrypted FS) cannot starve the event loop for the upload's + # Every filesystem call runs on a worker thread so slow storage (NFS, + # FUSE, encrypted FS) cannot starve the event loop for the upload's # duration. file_size = 0 - with open(target_path, "wb") as f: + f = await asyncio.to_thread(target_path.open, "wb") + try: while chunk := await file.read(8192): # Read in 8KB chunks await asyncio.to_thread(f.write, chunk) file_size += len(chunk) + finally: + await asyncio.to_thread(f.close) logger.info(f"Uploaded file to {target_path}") return FileUploadResult(file_size=file_size) @@ -133,7 +135,7 @@ async def _upload_file(path: str, file: UploadFile) -> FileUploadResult: raise OperationFailed(f"Failed to upload file: {str(e)}") -async def _download_file(path: str) -> FileResponse: +def _download_file(path: str) -> FileResponse: """Internal helper to download a file from the workspace.""" update_last_execution_time() logger.info(f"Downloading file: {path}") @@ -236,7 +238,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: try: await super().__call__(scope, receive, send) finally: - Path(self.path).unlink(missing_ok=True) + await asyncio.to_thread(Path(self.path).unlink, missing_ok=True) ArchiveFormat = Literal["git-delta", "tar.gz"] @@ -316,6 +318,52 @@ def _is_sensitive_git_internal(rel: PurePosixPath) -> bool: _ARCHIVE_DESCENT_SKIP = frozenset({"node_modules", "venv", "site-packages", "vendor"}) +def _existing_directory(target: Path) -> Path: + """``target`` resolved, or the 404/400 for a missing path or a non-directory.""" + if not target.exists(): + raise NotFound("Directory not found") + if not target.is_dir(): + raise Invalid("Path is not a directory") + return target.resolve() + + +def _archive_scratch_file(target: Path, suffix: str) -> Path: + """Create the empty file an archive of ``target`` is written to.""" + # Build the archive on the same volume as the workspace (its parent dir, so + # it is never included in itself) rather than the system temp dir: a large + # tar.gz on a tmpfs /tmp would defeat the stream-to-disk OOM fix, and on a + # small container root it could trip the pod's ephemeral-storage limit. Fall + # back to the default temp location if the parent is not usable. + scratch_dir: str | None = None + parent = target.parent + if parent != target and os.access(parent, os.W_OK): + scratch_dir = str(parent) + fd, tmp_name = tempfile.mkstemp(suffix=suffix, dir=scratch_dir) + os.close(fd) + return Path(tmp_name) + + +def _build_archive( + archive_format: ArchiveFormat, + repo_root: Path, + base_ref: str | None, + output_path: Path, + excludes: list[str], +) -> str: + """Write the archive to ``output_path``; the git-delta base commit or ``""``. + + ``output_path`` is removed when building fails. + """ + try: + if archive_format == "git-delta": + return _create_git_delta(repo_root, base_ref, output_path, excludes) + _create_tar_gz_archive(repo_root, output_path, excludes) + return "" + except Exception: + output_path.unlink(missing_ok=True) + raise + + def _resolve_git_repo_root(target: Path, max_depth: int = 3) -> Path: """Resolve the git work-tree to archive under ``target``. @@ -735,11 +783,11 @@ async def upload_file_query( @file_router.get( "/download", responses=_FILE_DOWNLOAD_RESPONSES, response_class=FileResponse ) -async def download_file_query( +def download_file_query( path: Annotated[str, Query(description="Absolute file path")], ) -> FileResponse: """Download a file from the workspace using query parameter (preferred method).""" - return await _download_file(path) + return _download_file(path) @file_router.post("/create_directory") @@ -848,7 +896,7 @@ async def get_home_directory( @file_discovery_router.get("/search_subdirs") -async def search_subdirs( +def search_subdirs( path: Annotated[ str, Query(description="Absolute directory path to list subdirectories of"), @@ -966,7 +1014,7 @@ async def download_trajectory( try: await zip_task finally: - temp_file.unlink(missing_ok=True) + await asyncio.to_thread(temp_file.unlink, missing_ok=True) raise return _TemporaryFileResponse( path=temp_file, @@ -1054,51 +1102,31 @@ async def archive_directory( target = Path(path) if not target.is_absolute(): raise Invalid("Path must be absolute") - if not target.exists(): - raise NotFound("Directory not found") - if not target.is_dir(): - raise Invalid("Path is not a directory") + target = await asyncio.to_thread(_existing_directory, target) if base_ref is not None and base_ref.startswith("-"): raise Invalid("base_ref must not start with '-'") - target = target.resolve() effective_excludes = ( list(_DEFAULT_ARCHIVE_EXCLUDES) if use_default_excludes else [] ) + (exclude or []) - # Build the archive on the same volume as the workspace (its parent dir, so - # it is never included in itself) rather than the system temp dir: a large - # tar.gz on a tmpfs /tmp would defeat the stream-to-disk OOM fix, and on a - # small container root it could trip the pod's ephemeral-storage limit. Fall - # back to the default temp location if the parent is not usable. - scratch_dir: str | None = None - parent = target.parent - if parent != target and os.access(parent, os.W_OK): - scratch_dir = str(parent) - fd, tmp_name = tempfile.mkstemp( - suffix=_ARCHIVE_SUFFIX[archive_format], dir=scratch_dir + output_path = await asyncio.to_thread( + _archive_scratch_file, target, _ARCHIVE_SUFFIX[archive_format] ) - os.close(fd) - output_path = Path(tmp_name) # The caller passes the workspace base, but a repo-backed conversation clones # into a subdirectory; resolve to the actual repo so both formats root the # archive at the repo (consistent paths across git-delta, tar.gz, and the # initial snapshot) and git-delta does not 400 on the non-repo parent. repo_root = await asyncio.to_thread(_resolve_git_repo_root, target) - base_commit = "" try: - if archive_format == "git-delta": - base_commit = await asyncio.to_thread( - _create_git_delta, - repo_root, - base_ref, - output_path, - effective_excludes, - ) - else: - await asyncio.to_thread( - _create_tar_gz_archive, repo_root, output_path, effective_excludes - ) + base_commit = await asyncio.to_thread( + _build_archive, + archive_format, + repo_root, + base_ref, + output_path, + effective_excludes, + ) except GitRepositoryError as e: output_path.unlink(missing_ok=True) raise Invalid(f"Not a git repository: {e}") @@ -1109,7 +1137,6 @@ async def archive_directory( output_path.unlink(missing_ok=True) raise OperationFailed(f"Failed to generate git delta: {e}") except Exception as e: - output_path.unlink(missing_ok=True) logger.error(f"Failed to archive {target}: {e}") raise OperationFailed(f"Failed to archive directory: {str(e)}") diff --git a/openhands-agent-server/openhands/agent_server/git_provider_service.py b/openhands-agent-server/openhands/agent_server/git_provider_service.py index 7d61adf6610..aae8e7a7723 100644 --- a/openhands-agent-server/openhands/agent_server/git_provider_service.py +++ b/openhands-agent-server/openhands/agent_server/git_provider_service.py @@ -8,7 +8,7 @@ import httpx -from openhands.agent_server.persistence import SecretsStore +from openhands.agent_server.persistence import SecretsStore, run_store from openhands.sdk.git.models import ( GitProviderRepository, GitProviderRepositoryPage, @@ -87,7 +87,7 @@ async def search_provider_repositories( f"Repository discovery is not supported for provider '{provider.value}'" ) - token = _resolve_provider_token(secrets_store, provider) + token = await run_store(_resolve_provider_token, secrets_store, provider) if token is None: return GitProviderRepositoryPage( items=[], next_page_id=None, missing_token=True @@ -121,7 +121,7 @@ async def get_provider_pull_request( raise UnsupportedGitProviderError( f"Pull requests are not supported for provider '{provider.value}'" ) - token = _resolve_provider_token(secrets_store, provider) + token = await run_store(_resolve_provider_token, secrets_store, provider) if token is None: raise MissingGitProviderTokenError(f"No {provider.value} token is configured") diff --git a/openhands-agent-server/openhands/agent_server/init_router.py b/openhands-agent-server/openhands/agent_server/init_router.py index 905da480cc7..d04740abdfb 100644 --- a/openhands-agent-server/openhands/agent_server/init_router.py +++ b/openhands-agent-server/openhands/agent_server/init_router.py @@ -44,7 +44,11 @@ ) from openhands.agent_server.conversation_service import ConversationService from openhands.agent_server.dependencies import get_config -from openhands.agent_server.persistence import ServerStores, reencrypt_stale_stores +from openhands.agent_server.persistence import ( + ServerStores, + reencrypt_stale_stores, + run_store, +) from openhands.agent_server.server_context import ( ServerContext, install_server_context, @@ -337,7 +341,7 @@ async def _prepare(self, req: InitRequest) -> _PreparedInit: maybe_init_laminar() stores = ServerStores.from_config(new_config) - await asyncio.to_thread(reencrypt_stale_stores, stores) + await run_store(reencrypt_stale_stores, stores) # The conversation service reads the sink while it starts, so the # delivered one has to be in place first. ``server_started`` is diff --git a/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py b/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py index f463c84b970..dcb47007cdb 100644 --- a/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py +++ b/openhands-agent-server/openhands/agent_server/mcp_oauth_store.py @@ -8,7 +8,6 @@ from __future__ import annotations -import asyncio import copy import threading from collections.abc import Mapping, Sequence @@ -18,7 +17,11 @@ import httpx from openhands.agent_server.config import Config, WebhookSpec -from openhands.agent_server.persistence import PersistedSettings, SettingsStore +from openhands.agent_server.persistence import ( + PersistedSettings, + SettingsStore, + run_store, +) from openhands.sdk.logger import get_logger from openhands.sdk.mcp.client import MCPClient from openhands.sdk.mcp.config import ( @@ -167,13 +170,13 @@ def _get_entry_sync( async def get( self, key: str, *, collection: str | None = None ) -> dict[str, Any] | None: - value, _ = await asyncio.to_thread(self._get_entry_sync, key, collection) + value, _ = await run_store(self._get_entry_sync, key, collection) return value async def ttl( self, key: str, *, collection: str | None = None ) -> tuple[dict[str, Any] | None, float | None]: - return await asyncio.to_thread(self._get_entry_sync, key, collection) + return await run_store(self._get_entry_sync, key, collection) def _put_sync( self, @@ -285,7 +288,7 @@ async def put( collection: str | None = None, ttl: SupportsFloat | None = None, ) -> None: - await asyncio.to_thread( + await run_store( self._put_sync, key, value, @@ -335,7 +338,7 @@ def apply_update(settings: PersistedSettings) -> PersistedSettings: return deleted async def delete(self, key: str, *, collection: str | None = None) -> bool: - return await asyncio.to_thread(self._delete_sync, key, collection) + return await run_store(self._delete_sync, key, collection) async def get_many( self, keys: Sequence[str], *, collection: str | None = None diff --git a/openhands-agent-server/openhands/agent_server/openai/service.py b/openhands-agent-server/openhands/agent_server/openai/service.py index 0095309a569..d6a85da0f21 100644 --- a/openhands-agent-server/openhands/agent_server/openai/service.py +++ b/openhands-agent-server/openhands/agent_server/openai/service.py @@ -43,7 +43,11 @@ OpenAIResponseUsage, OpenAIUsage, ) -from openhands.agent_server.persistence import PersistedSettings, ServerStores +from openhands.agent_server.persistence import ( + PersistedSettings, + ServerStores, + run_store, +) from openhands.sdk import Message from openhands.sdk.conversation.request import ( AgentLaunchAdditions, @@ -508,7 +512,7 @@ def iter_openai_chat_completion_sse( async def list_openai_models(stores: ServerStores) -> OpenAIModelListResponse: try: - profiles = await asyncio.to_thread(stores.llm_profiles.list_summaries) + profiles = await run_store(stores.llm_profiles.list_summaries) except TimeoutError: raise Unavailable("Profile store is busy. Please retry.") data = [ @@ -762,7 +766,9 @@ async def run_chat_completion( # SSE streaming needs incremental agent-event forwarding; add it separately. raise Invalid("Streaming chat completions are not supported yet") - start_request = _conversation_request( + # Reads the profile and settings stores. + start_request = await run_store( + _conversation_request, request=request, config=config, stores=stores, @@ -826,7 +832,9 @@ async def run_response( ) user_content = _response_user_content(request) - start_request = _response_conversation_request( + # Reads the profile and settings stores. + start_request = await run_store( + _response_conversation_request, request=request, user_content=user_content, config=config, diff --git a/openhands-agent-server/openhands/agent_server/persistence/__init__.py b/openhands-agent-server/openhands/agent_server/persistence/__init__.py index aaaa11b9c6f..17d12f10a94 100644 --- a/openhands-agent-server/openhands/agent_server/persistence/__init__.py +++ b/openhands-agent-server/openhands/agent_server/persistence/__init__.py @@ -6,6 +6,7 @@ See: openhands.sdk.settings.api_models """ +from openhands.agent_server.persistence.async_store import run_store from openhands.agent_server.persistence.models import ( PERSISTED_SETTINGS_SCHEMA_VERSION, SECRET_NAME_PATTERN, @@ -50,6 +51,8 @@ "SettingsUpdatePayload", "WorkspaceItem", "WorkspaceParentItem", + # Async access + "run_store", # Stores "FileSecretsStore", "FileSettingsStore", diff --git a/openhands-agent-server/openhands/agent_server/persistence/async_store.py b/openhands-agent-server/openhands/agent_server/persistence/async_store.py new file mode 100644 index 00000000000..7d2c511fe49 --- /dev/null +++ b/openhands-agent-server/openhands/agent_server/persistence/async_store.py @@ -0,0 +1,36 @@ +"""The one way async code reaches a persistence store. + +The stores (settings, secrets, workspaces, LLM/agent/meta profiles, provider +connections, MCP OAuth tokens) are synchronous: each call reads or writes +files and may wait up to its lock timeout for another writer. Called from an +``async def`` they would stall the event loop, and with it every other +request, websocket and background task, for that whole wait. + +``run_store`` runs a store call, or a sync function doing several store +calls, on a worker thread and awaits it. Keep a read-modify-write in one +function so it makes one hop:: + + settings = await run_store(store.load) + + def add(workspaces: PersistedWorkspaces) -> PersistedWorkspaces: ... + await run_store(store.update, add) + +``scripts/check_async_store_access.py`` (pre-commit) rejects store calls made +directly in async code or hopped with ``asyncio.to_thread`` / +``run_in_executor`` instead of this function. +""" + +import asyncio +from collections.abc import Callable + + +async def run_store[**P, R]( + fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs +) -> R: + """Run blocking store work ``fn(*args, **kwargs)`` off the event loop. + + Uses the loop's default executor and copies the caller's context + variables (``asyncio.to_thread`` semantics). Cancelling the awaiting task + does not stop a write already running on the worker thread. + """ + return await asyncio.to_thread(fn, *args, **kwargs) diff --git a/openhands-agent-server/openhands/agent_server/profiles_router.py b/openhands-agent-server/openhands/agent_server/profiles_router.py index cb91a773395..0ea41503513 100644 --- a/openhands-agent-server/openhands/agent_server/profiles_router.py +++ b/openhands-agent-server/openhands/agent_server/profiles_router.py @@ -24,6 +24,7 @@ FileSettingsStore, PersistedSettings, ProviderConnectionStore, + run_store, ) from openhands.sdk.errors import Conflict, NotFound, OperationFailed from openhands.sdk.llm import LLM, Message, TextContent @@ -352,7 +353,7 @@ async def validate_profile( # Check a draft linked to a provider connection with the connection's # credentials, as activation and ``switch_llm`` run it. No-op without # ``provider_connection_id``; a dangling link is a 422 naming it. - llm = store.resolve_provider_connection(llm, cipher=cipher) + llm = await run_store(store.resolve_provider_connection, llm, cipher=cipher) messages = [ Message( diff --git a/openhands-agent-server/openhands/agent_server/runtime_router.py b/openhands-agent-server/openhands/agent_server/runtime_router.py index a5f7a394276..f2cae174808 100644 --- a/openhands-agent-server/openhands/agent_server/runtime_router.py +++ b/openhands-agent-server/openhands/agent_server/runtime_router.py @@ -4,6 +4,7 @@ checks are routing safeguards, not a sandbox for arbitrary shell commands. """ +import asyncio from collections.abc import Callable from copy import deepcopy from pathlib import Path @@ -46,14 +47,11 @@ async def bind_local_conversation_runtime( if event_service is None: raise NotFound("Conversation not found") request.state.runtime_event_service = event_service - workspace_root = Path( - event_service.get_conversation().workspace.working_dir - ).resolve() + workspace_dir = event_service.get_conversation().workspace.working_dir for name in ("path", "workspace_dir"): requested_path = request.path_params.get(name) or request.query_params.get(name) - if requested_path is not None and ( - not Path(requested_path).is_absolute() - or not Path(requested_path).resolve().is_relative_to(workspace_root) + if requested_path is not None and not await asyncio.to_thread( + _is_inside, Path(requested_path), Path(workspace_dir) ): raise Unprocessable(f"{name} must be inside the conversation workspace") trajectory_id = request.path_params.get("conversation_id") @@ -66,6 +64,11 @@ async def bind_local_conversation_runtime( raise Unprocessable("Trajectory must belong to the selected conversation") +def _is_inside(path: Path, workspace_dir: Path) -> bool: + """Whether absolute ``path`` resolves inside ``workspace_dir``.""" + return path.is_absolute() and path.resolve().is_relative_to(workspace_dir.resolve()) + + class RuntimeRouter(APIRouter): def add_api_route( self, path: str, endpoint: Callable[..., Any], **kwargs: Any diff --git a/openhands-agent-server/openhands/agent_server/settings_router.py b/openhands-agent-server/openhands/agent_server/settings_router.py index 7f951d2c7b4..7d70d9e98d0 100644 --- a/openhands-agent-server/openhands/agent_server/settings_router.py +++ b/openhands-agent-server/openhands/agent_server/settings_router.py @@ -25,6 +25,7 @@ FileSettingsStore, PersistedSettings, ServerStores, + run_store, ) from openhands.agent_server.persistence.models import SettingsUpdatePayload from openhands.agent_server.telemetry import notify_misc_settings_changed @@ -168,7 +169,7 @@ async def get_settings( values. """ expose_mode = parse_expose_secrets_header(request) - settings = store.load() or PersistedSettings() + settings = await run_store(store.load) or PersistedSettings() # Audit log all settings access for security visibility # Use WARNING level for plaintext mode to highlight security-sensitive operations @@ -452,7 +453,8 @@ async def create_mcp_server( SettingsUpdatePayload, {"agent_settings_diff": {"mcp_config": {settings_key: server_data}}}, ) - return _apply_settings_update( + return await run_store( + _apply_settings_update, request, config, stores, @@ -479,7 +481,8 @@ async def patch_mcp_server( SettingsUpdatePayload, {"agent_settings_diff": {"mcp_config": {settings_key: patch_data}}}, ) - return _apply_settings_update( + return await run_store( + _apply_settings_update, request, config, stores, @@ -500,7 +503,8 @@ async def delete_mcp_server( SettingsUpdatePayload, {"agent_settings_diff": {"mcp_config": {settings_key: None}}}, ) - return _apply_settings_update( + return await run_store( + _apply_settings_update, request, config, stores, @@ -568,7 +572,7 @@ async def get_secret_value( """ _validate_secret_name(name) - value = store.get_secret(name) + value = await run_store(store.get_secret, name) client_host = request.client.host if request.client else "unknown" if value is None: @@ -603,7 +607,8 @@ async def create_secret( value = secret.value.get_secret_value() try: - store.set_secret( + await run_store( + store.set_secret, name=secret.name, value=value, description=secret.description, @@ -647,7 +652,7 @@ async def delete_secret( client_host = request.client.host if request.client else "unknown" try: - deleted = store.delete_secret(name) + deleted = await run_store(store.delete_secret, name) except RuntimeError as e: # Data corruption protection triggered (file exists but unreadable) logger.error(f"Secret delete blocked: {e}") diff --git a/openhands-agent-server/openhands/agent_server/telemetry/service.py b/openhands-agent-server/openhands/agent_server/telemetry/service.py index 91eabc3fc5e..7280938752d 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/service.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/service.py @@ -8,11 +8,11 @@ sink. Analytics must never be able to prevent a server from starting. """ -import asyncio from functools import partial from typing import Any from openhands.agent_server.config import Config +from openhands.agent_server.persistence.async_store import run_store from openhands.agent_server.persistence.store import SettingsStore from openhands.agent_server.telemetry.factory import ( DiagnosticEventFactory, @@ -46,7 +46,7 @@ def _read_misc_settings_sync(settings_store: SettingsStore) -> dict[str, Any]: async def _read_decision(settings_store: SettingsStore) -> TelemetryDecision: """Resolve consent from persisted settings, off the event loop.""" - misc = await asyncio.to_thread(_read_misc_settings_sync, settings_store) + misc = await run_store(_read_misc_settings_sync, settings_store) return resolve(misc) diff --git a/openhands-agent-server/openhands/agent_server/vscode_service.py b/openhands-agent-server/openhands/agent_server/vscode_service.py index 6a8b7c58d83..a8b4d33979e 100644 --- a/openhands-agent-server/openhands/agent_server/vscode_service.py +++ b/openhands-agent-server/openhands/agent_server/vscode_service.py @@ -211,7 +211,8 @@ async def _wait_for_exit(self) -> None: if self.process is None: return await self.process.wait() - while not await self._is_port_available(): + # Nothing signals that node released the port; polling is the only way. + while not await self._is_port_available(): # noqa: ASYNC110 await asyncio.sleep(0.1) def _proxy_script_path(self) -> Path: diff --git a/openhands-agent-server/openhands/agent_server/workspace_router.py b/openhands-agent-server/openhands/agent_server/workspace_router.py index 4d37e48f0fa..de0f711fa0a 100644 --- a/openhands-agent-server/openhands/agent_server/workspace_router.py +++ b/openhands-agent-server/openhands/agent_server/workspace_router.py @@ -12,6 +12,7 @@ - Path traversal outside of the workspace is rejected. """ +import asyncio from pathlib import Path from uuid import UUID @@ -50,8 +51,8 @@ async def _resolve_workspace_dir( workspace = event_service.stored.workspace if not isinstance(workspace, LocalWorkspace): raise NotFound("Conversation workspace is not local; cannot be served") - workspace_dir = Path(workspace.working_dir).resolve() - if not workspace_dir.is_dir(): + workspace_dir = await asyncio.to_thread(Path(workspace.working_dir).resolve) + if not await asyncio.to_thread(workspace_dir.is_dir): raise NotFound("Workspace directory does not exist") return workspace_dir diff --git a/openhands-agent-server/openhands/agent_server/workspaces_router.py b/openhands-agent-server/openhands/agent_server/workspaces_router.py index b41fd7b63f3..7ede0cc4ab0 100644 --- a/openhands-agent-server/openhands/agent_server/workspaces_router.py +++ b/openhands-agent-server/openhands/agent_server/workspaces_router.py @@ -17,6 +17,7 @@ PersistedWorkspaces, WorkspaceItem, WorkspaceParentItem, + run_store, ) from openhands.sdk.errors import Conflict, NotFound, OperationFailed from openhands.sdk.logger import get_logger @@ -65,17 +66,22 @@ async def list_workspaces( store: FileWorkspacesStore = Depends(get_workspaces_store), ) -> WorkspacesListResponse: """Return the saved workspaces and workspace parents.""" - try: + + def read() -> tuple[PersistedWorkspaces | None, bool]: persisted = store.load() - except (OSError, PermissionError): - logger.error("Workspaces list failed - file I/O error") - raise OperationFailed("Failed to read workspaces") - if persisted is None: # ``load()`` collapses "file missing" and "file present but # unreadable/corrupted/future-schema" into the same ``None``. Returning # an empty list for the second case would hide the user's data and let # a subsequent POST overwrite the still-on-disk file with defaults. - if store._path.exists(): + return persisted, persisted is None and store._path.exists() + + try: + persisted, corrupted = await run_store(read) + except (OSError, PermissionError): + logger.error("Workspaces list failed - file I/O error") + raise OperationFailed("Failed to read workspaces") + if persisted is None: + if corrupted: raise Conflict("Workspaces file is corrupted") persisted = PersistedWorkspaces() return _to_response(persisted) @@ -110,7 +116,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: ) try: - updated = store.update(apply) + updated = await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspaces add blocked: {e}") raise Conflict("Workspaces file is corrupted") @@ -139,7 +145,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: return current.model_copy(update={"workspaces": remaining}) try: - store.update(apply) + await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspaces delete blocked: {e}") raise Conflict("Workspaces file is corrupted") @@ -182,7 +188,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: ) try: - updated = store.update(apply) + updated = await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspace parents add blocked: {e}") raise Conflict("Workspaces file is corrupted") @@ -211,7 +217,7 @@ def apply(current: PersistedWorkspaces) -> PersistedWorkspaces: return current.model_copy(update={"workspace_parents": remaining}) try: - store.update(apply) + await run_store(store.update, apply) except RuntimeError as e: logger.error(f"Workspace parents delete blocked: {e}") raise Conflict("Workspaces file is corrupted") diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index f83d1726560..061a9a621ba 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -1348,7 +1348,9 @@ async def wrapped(*args: Any, **kwargs: Any) -> _T: return decorate async def _anext_with_idle_timeout( - self, iterator: AsyncIterator[_T], timeout: float + self, + iterator: AsyncIterator[_T], + timeout: float, # noqa: ASYNC109 - this function is the cancel scope ) -> _T: """Await one chunk, converting only this idle timer's own expiry. diff --git a/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py b/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py index 796d629da1a..bb25709870a 100644 --- a/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py +++ b/openhands-sdk/openhands/sdk/workspace/remote/async_remote_workspace.py @@ -78,7 +78,7 @@ async def start_command( self, command: str, cwd: str | Path | None = None, - timeout: float = 30, + timeout: float = 30, # noqa: ASYNC109 - sent to the server's command ) -> str: """Start a command and return its ID without waiting for completion.""" return await self._execute(self._start_command_generator(command, cwd, timeout)) @@ -109,7 +109,7 @@ async def execute_command( self, command: str, cwd: str | Path | None = None, - timeout: float = 30.0, + timeout: float = 30.0, # noqa: ASYNC109 - sent to the server's command ) -> CommandResult: """Execute a bash command on the remote system. diff --git a/pyproject.toml b/pyproject.toml index 0a1971ae8af..b775cf257ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ select = [ "I", # isort "UP", # pyupgrade "ARG", # flake8-unused-arguments + "ASYNC", # flake8-async: no blocking calls on the event loop ] # Enforce rules that catch mutable defaults and related pitfalls # - B006: mutable-argument-default @@ -78,7 +79,8 @@ ignore = ["UP042"] [tool.ruff.lint.per-file-ignores] # Test files often have unused arguments (fixtures, mocks, interface implementations) -"tests/**/*.py" = ["ARG"] +# and poll or block on purpose while driving threads and subprocesses. +"tests/**/*.py" = ["ARG", "ASYNC"] # Allowlist safe default calls for flake8-bugbear rules (e.g., FastAPI Depends) diff --git a/scripts/check_async_store_access.py b/scripts/check_async_store_access.py new file mode 100644 index 00000000000..b35d0942e3b --- /dev/null +++ b/scripts/check_async_store_access.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Keep blocking store I/O off the agent-server event loop. + +The persistence stores (settings, secrets, workspaces, LLM/agent/meta +profiles, provider connections, MCP OAuth tokens, runtime provisioning) are +synchronous: they read and write files under a file lock. Inside an ``async +def`` they must be reached through ``run_store`` from +``openhands.agent_server.persistence.async_store``, the one place where store +I/O hops to a worker thread. + +This check flags, inside every ``async def`` under ``openhands-agent-server``: + +1. a direct method call on a store (``store.load()``, + ``get_settings_store(config).update(fn)``, ``registry.provisioning.save(x)``); +2. store work handed to any other thread hop (``asyncio.to_thread``, + ``loop.run_in_executor``, ``run_in_threadpool``) instead of ``run_store``: + a store method as the hopped callable, or a lambda that calls one. + +A store expression is: + +- a local name annotated with a store type or assigned from a store getter or + constructor, or a getter call used inline; +- an attribute (``self.x`` or ``obj.x``) whose name some agent-server class + declares with a store type or assigns from a store getter or constructor. + Attribute names are matched across classes, so give a store attribute a + name no non-store attribute uses. + +Code in nested sync ``def``s and lambdas is not checked: those run wherever +their caller puts them, normally inside ``run_store``. + +Limits: a store held in a container or reached through an unannotated +parameter is not seen, and a sync helper that does store I/O and is called +directly from async code is not followed. Methods in ``_PURE_METHODS`` touch +no file and are allowed. + +Usage: check_async_store_access.py [FILES...] +(no files: scan openhands-agent-server) +""" + +import ast +import sys +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCAN_ROOT = ROOT / "openhands-agent-server" / "openhands" / "agent_server" + +STORE_TYPES = frozenset( + { + "SettingsStore", + "FileSettingsStore", + "SecretsStore", + "FileSecretsStore", + "WorkspacesStore", + "FileWorkspacesStore", + "LLMProfileStore", + "AgentProfileStore", + "MetaProfileStore", + "ProviderConnectionStore", + "MCPSettingsOAuthTokenStore", + "RuntimeProvisioningStore", + } +) + +STORE_GETTERS = frozenset( + { + "get_settings_store", + "get_secrets_store", + "get_workspaces_store", + "get_provider_connections_store", + "get_llm_profile_store", + "get_agent_profile_store", + } +) + +# Methods that only build or inspect an object and touch no file. +_PURE_METHODS = frozenset({"with_cipher", "manifest_path"}) + +THREAD_HOPS = frozenset({"to_thread", "run_in_executor", "run_in_threadpool"}) +FACADE = "run_store" + +_NESTED_SCOPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef) + + +@dataclass(frozen=True) +class Violation: + path: Path + line: int + message: str + + def __str__(self) -> str: + try: + shown = self.path.relative_to(ROOT) + except ValueError: + shown = self.path + return f"{shown}:{self.line}: {self.message}" + + +def _annotation_names(node: ast.expr | None) -> Iterator[str]: + """Every bare or dotted type name mentioned in an annotation.""" + if node is None: + return + for sub in ast.walk(node): + if isinstance(sub, ast.Name): + yield sub.id + elif isinstance(sub, ast.Attribute): + yield sub.attr + elif isinstance(sub, ast.Constant) and isinstance(sub.value, str): + try: + parsed = ast.parse(sub.value, mode="eval").body + except SyntaxError: + continue + yield from _annotation_names(parsed) + + +def _is_store_annotation(node: ast.expr | None) -> bool: + return any(name in STORE_TYPES for name in _annotation_names(node)) + + +def _callee_name(call: ast.Call) -> str | None: + if isinstance(call.func, ast.Name): + return call.func.id + if isinstance(call.func, ast.Attribute): + return call.func.attr + return None + + +def _makes_store(value: ast.expr | None) -> bool: + """True for ``get_x_store(...)`` or ``XStore(...)`` (possibly awaited).""" + if isinstance(value, ast.Await): + value = value.value + if not isinstance(value, ast.Call): + return False + name = _callee_name(value) + return name in STORE_GETTERS or name in STORE_TYPES + + +def store_attributes(tree: ast.AST) -> set[str]: + """Attribute names that classes in ``tree`` use to hold a store.""" + attrs: set[str] = set() + for cls in ast.walk(tree): + if not isinstance(cls, ast.ClassDef): + continue + for node in ast.walk(cls): + if isinstance(node, ast.AnnAssign): + target = node.target + if isinstance(target, ast.Name | ast.Attribute) and ( + _is_store_annotation(node.annotation) or _makes_store(node.value) + ): + attrs.add( + target.id if isinstance(target, ast.Name) else target.attr + ) + elif isinstance(node, ast.Assign) and _makes_store(node.value): + attrs.update( + target.attr + for target in node.targets + if isinstance(target, ast.Attribute) + ) + return attrs + + +def _own_nodes(func: ast.AsyncFunctionDef) -> Iterator[ast.AST]: + """Nodes of ``func``'s body, not descending into nested scopes.""" + stack: list[ast.AST] = [ + stmt for stmt in func.body if not isinstance(stmt, _NESTED_SCOPES) + ] + while stack: + node = stack.pop() + yield node + stack.extend( + child + for child in ast.iter_child_nodes(node) + if not isinstance(child, _NESTED_SCOPES) + ) + + +class _AsyncFunctionChecker: + def __init__( + self, path: Path, func: ast.AsyncFunctionDef, store_attrs: set[str] + ) -> None: + self.path = path + self.func = func + self.store_attrs = store_attrs + self.store_names = self._collect_store_names() + + def _collect_store_names(self) -> set[str]: + names: set[str] = set() + args = self.func.args + for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs]: + if _is_store_annotation(arg.annotation): + names.add(arg.arg) + for node in _own_nodes(self.func): + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if _is_store_annotation(node.annotation) or _makes_store(node.value): + names.add(node.target.id) + elif isinstance(node, ast.Assign) and _makes_store(node.value): + for target in node.targets: + if isinstance(target, ast.Name): + names.add(target.id) + return names + + def _is_store(self, node: ast.expr) -> bool: + if isinstance(node, ast.Name): + return node.id in self.store_names + if isinstance(node, ast.Attribute): + return node.attr in self.store_attrs + return _makes_store(node) + + def _store_method(self, node: ast.expr) -> str | None: + """``m`` when ``node`` is ``.m`` for a method that does I/O.""" + if ( + isinstance(node, ast.Attribute) + and node.attr not in _PURE_METHODS + and self._is_store(node.value) + ): + return node.attr + return None + + def _hopped_store_work(self, call: ast.Call, callee: str) -> str | None: + """The store method a thread hop runs, if its callable is store work. + + Only the callable position counts (``run_in_executor`` takes the + executor first); a store method passed further along is a callback + for the hopped function, not the hop's own work. + """ + position = 1 if callee == "run_in_executor" else 0 + if len(call.args) <= position: + return None + hopped = call.args[position] + if isinstance(hopped, ast.Lambda): + for sub in ast.walk(hopped.body): + if isinstance(sub, ast.Call): + method = self._store_method(sub.func) + if method is not None: + return method + return None + return self._store_method(hopped) + + def violations(self) -> Iterator[Violation]: + for node in _own_nodes(self.func): + if not isinstance(node, ast.Call): + continue + method = self._store_method(node.func) + if method is not None: + yield Violation( + self.path, + node.lineno, + f"blocking store call .{method}() in async " + f"{self.func.name}(); await {FACADE}(...) instead", + ) + continue + callee = _callee_name(node) + if callee not in THREAD_HOPS: + continue + hopped = self._hopped_store_work(node, callee) + if hopped is not None: + yield Violation( + self.path, + node.lineno, + f"store work .{hopped} handed to {callee}() in " + f"async {self.func.name}(); use {FACADE}(...)", + ) + + +def check_source( + path: Path, source: str, store_attrs: Iterable[str] = () +) -> list[Violation]: + """Violations in one module; ``store_attrs`` adds attribute names declared + elsewhere (the module's own classes are always read).""" + tree = ast.parse(source, filename=str(path)) + attrs = store_attributes(tree) | set(store_attrs) + found = [ + violation + for node in ast.walk(tree) + if isinstance(node, ast.AsyncFunctionDef) + for violation in _AsyncFunctionChecker(path, node, attrs).violations() + ] + return sorted(found, key=lambda v: v.line) + + +def _targets(argv: list[str]) -> Iterable[Path]: + if not argv: + return sorted(SCAN_ROOT.rglob("*.py")) + paths = [Path(arg).resolve() for arg in argv] + return [p for p in paths if p.suffix == ".py" and p.is_relative_to(SCAN_ROOT)] + + +def main(argv: list[str]) -> int: + # Store attributes are collected from the whole package even when only + # some files are checked: ``registry.provisioning`` is declared elsewhere. + sources = { + path: path.read_text(encoding="utf-8") for path in SCAN_ROOT.rglob("*.py") + } + store_attrs: set[str] = set() + for source in sources.values(): + store_attrs |= store_attributes(ast.parse(source)) + violations: list[Violation] = [] + for path in _targets(argv): + source = sources.get(path) or path.read_text(encoding="utf-8") + violations.extend(check_source(path, source, store_attrs)) + for violation in violations: + print(violation) + if violations: + print( + f"\n{len(violations)} blocking store access(es) in async code. " + f"Wrap the store work in a sync function and await {FACADE}(fn, ...)." + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/agent_server/stress/test_slow_webhook.py b/tests/agent_server/stress/test_slow_webhook.py index 6caf5ed0767..6df16c2a3ce 100644 --- a/tests/agent_server/stress/test_slow_webhook.py +++ b/tests/agent_server/stress/test_slow_webhook.py @@ -188,7 +188,7 @@ async def test_slow_webhook_does_not_unbound_growth( (b) the webhook subscriber buffer doesn't accumulate unbounded events. """ # Distinct workspaces per run so any workspace-side state from the - # baseline (e.g. .git from `_ensure_workspace_is_git_repo`, scratch + # baseline (e.g. .git from `_prepare_workspace_dir`, scratch # files) doesn't bleed into the webhook timing. workspace_baseline = str(tmp_path / "ws_baseline") workspace_webhook = str(tmp_path / "ws_webhook") diff --git a/tests/agent_server/test_profile_store_lock_responsiveness.py b/tests/agent_server/test_profile_store_lock_responsiveness.py index ffef832c82d..d6d57d14198 100644 --- a/tests/agent_server/test_profile_store_lock_responsiveness.py +++ b/tests/agent_server/test_profile_store_lock_responsiveness.py @@ -71,6 +71,30 @@ 200, id="provider-connections", ), + pytest.param( + ".workspaces.lock", + "POST", + "/api/workspaces", + {"workspaces": [{"id": "/qa-busy", "name": "qa", "path": "/qa-busy"}]}, + 200, + id="workspaces", + ), + pytest.param( + ".secrets.lock", + "PUT", + "/api/settings/secrets", + {"name": "QA_BUSY", "value": "qa-value"}, + 200, + id="secrets", + ), + pytest.param( + ".settings.lock", + "POST", + "/api/settings/mcp/qa-busy", + {"transport": "http", "url": "https://qa-busy.example/mcp"}, + 201, + id="settings-mcp-server", + ), ], ) async def test_alive_answers_while_store_request_waits_for_lock( diff --git a/tests/cross/test_check_async_store_access.py b/tests/cross/test_check_async_store_access.py new file mode 100644 index 00000000000..3a73667a2ce --- /dev/null +++ b/tests/cross/test_check_async_store_access.py @@ -0,0 +1,143 @@ +"""Tests for the pre-commit check that keeps store I/O off the event loop.""" + +import importlib.util +import textwrap +from pathlib import Path +from types import ModuleType + +import pytest + + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "check_async_store_access.py" + + +@pytest.fixture(scope="module") +def checker() -> ModuleType: + spec = importlib.util.spec_from_file_location("check_async_store_access", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _lines(checker: ModuleType, source: str) -> list[int]: + violations = checker.check_source(Path("mod.py"), textwrap.dedent(source)) + return [violation.line for violation in violations] + + +@pytest.mark.parametrize( + "source", + [ + pytest.param( + """ + async def route(config): + store = get_settings_store(config) + return store.load() + """, + id="local-from-getter", + ), + pytest.param( + """ + async def route(config): + return get_secrets_store(config).get_secret("X") + """, + id="inline-getter", + ), + pytest.param( + """ + async def route(store: FileWorkspacesStore | None): + store.update(lambda w: w) + """, + id="annotated-parameter", + ), + pytest.param( + """ + class Registry: + provisioning: RuntimeProvisioningStore + + async def route(registry): + return registry.provisioning.load(1) + """, + id="attribute-through-other-object", + ), + pytest.param( + """ + async def route(store: LLMProfileStore): + return await asyncio.to_thread(store.list_summaries) + """, + id="to-thread-hop", + ), + pytest.param( + """ + async def route(config): + return await asyncio.to_thread( + lambda: get_settings_store(config).load() or None + ) + """, + id="lambda-hop", + ), + pytest.param( + """ + async def route(loop, store: FileSecretsStore): + return await loop.run_in_executor(None, store.get_secret, "X") + """, + id="run-in-executor-hop", + ), + ], +) +def test_flags_store_io_on_the_event_loop(checker: ModuleType, source: str) -> None: + assert len(_lines(checker, source)) == 1 + + +@pytest.mark.parametrize( + "source", + [ + pytest.param( + """ + async def route(config): + store = get_settings_store(config) + return await run_store(store.load) + """, + id="facade", + ), + pytest.param( + """ + async def route(store: FileWorkspacesStore): + def read(): + return store.load(), store.load() + return await run_store(read) + """, + id="nested-sync-helper", + ), + pytest.param( + """ + def route(config): + return get_settings_store(config).load() + """, + id="sync-route", + ), + pytest.param( + """ + async def route(registry: Registry, store: LLMProfileStore): + path = registry.provisioning.manifest_path(1) + return store.with_cipher(None), path + """, + id="pure-methods", + ), + pytest.param( + """ + async def route(loop, store: FileSecretsStore): + return await loop.run_in_executor(None, probe, store.get_secret) + """, + id="store-method-as-callback", + ), + ], +) +def test_allows_store_io_off_the_event_loop(checker: ModuleType, source: str) -> None: + assert _lines(checker, source) == [] + + +def test_agent_server_has_no_blocking_store_access( + checker: ModuleType, capsys: pytest.CaptureFixture[str] +) -> None: + assert checker.main([]) == 0, capsys.readouterr().out