diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py index c41098d4438..fe6c12b18a0 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py @@ -9,8 +9,8 @@ forces ``enabled=False`` as an explicit post-write step. Staged refresh -- ``check``/``apply`` replace ``update()``/ -``install(force=True)``, which rmtree+copytree straight onto the live -install path with no staging directory. ``check`` fetches and validates +``install(force=True)``, which fetch and swap in one step, with nothing to +confirm in between. ``check`` fetches and validates into ``.staging/`` without touching the active install; ``apply`` swaps it in via two atomic renames (POSIX can't atomically replace a non-empty directory in one rename), rolling back if the second fails. @@ -18,6 +18,7 @@ import os import shutil +import tempfile from contextlib import suppress from pathlib import Path @@ -34,12 +35,14 @@ InstallationInfo, InstallationInterface, InstallationManager, - InstallationMetadata, ) from openhands.sdk.extensions.installation.manager import DEFAULT_CACHE_DIR +from openhands.sdk.logger import get_logger from openhands.sdk.utils.path import get_user_persistence_dir +logger = get_logger(__name__) + # Matches the InstalledPluginInfo naming convention. InstalledCanvasExtensionInfo = InstallationInfo @@ -78,19 +81,6 @@ def _manager(installed_dir: Path) -> InstallationManager[CanvasExtensionManifest ) -def _tracked_names(installed_dir: Path) -> set[str]: - """Names with a real, valid tracked entry, not just a metadata key. - - A stale record with no matching directory must not count as "already - installed" -- otherwise a real install of that name would wrongly - inherit its state as if this were a force-reinstall. - """ - if not installed_dir.exists(): - return set() - metadata = InstallationMetadata.load_from_dir(installed_dir) - return {info.name for info in metadata.validate_tracked(installed_dir)} - - def _force_disable_new( manager: InstallationManager[CanvasExtensionManifest], info: InstallationInfo, @@ -132,9 +122,14 @@ def install_canvas_extension( """ installed_dir = _resolve_installed_dir(installed_dir) manager = _manager(installed_dir) - pre_existing = _tracked_names(installed_dir) - info = manager.install(source, ref=ref, repo_path=repo_path, force=force) - return _force_disable_new(manager, info, pre_existing) + # Whether this replaced an installed extension is decided inside the + # install's own metadata session: a snapshot taken before the install + # can be made stale by a concurrent uninstall. + outcome = manager.install_with_outcome( + source, ref=ref, repo_path=repo_path, force=force + ) + pre_existing = {outcome.info.name} if outcome.was_installed else set() + return _force_disable_new(manager, outcome.info, pre_existing) def uninstall_canvas_extension(name: str, installed_dir: Path | None = None) -> bool: @@ -172,9 +167,16 @@ def list_installed_canvas_extensions( discovered this way also land disabled. """ installed_dir = _resolve_installed_dir(installed_dir) + if not installed_dir.exists(): + return [] manager = _manager(installed_dir) - pre_existing = _tracked_names(installed_dir) - infos = manager.list_installed() + # Decided under the lock the listing takes: a reinstall swapping the + # directory in can't make an installed extension look new meanwhile. + with manager.metadata_session() as session: + pre_existing = { + info.name for info in session.metadata.validate_tracked(installed_dir) + } + infos = session.sync() return [_force_disable_new(manager, info, pre_existing) for info in infos] @@ -422,8 +424,7 @@ def apply_canvas_extension_update( """ installed_dir = _resolve_installed_dir(installed_dir) manager = _manager(installed_dir) - current_info = manager.get(name) - if current_info is None: + if manager.get(name) is None: return None staged_path = _staged_path(installed_dir, name, resolved_ref) @@ -435,31 +436,50 @@ def apply_canvas_extension_update( manifest = _load_validated_manifest(staged_path, name) install_path = installed_dir / name - backup_path = _staging_root(installed_dir) / f"{name}.previous" - shutil.rmtree(backup_path, ignore_errors=True) - - # POSIX rename() can't atomically replace a non-empty directory, so the - # active bundle moves aside first; roll back if the second rename fails. - os.replace(install_path, backup_path) - try: - os.replace(staged_path, install_path) - except OSError: - os.replace(backup_path, install_path) - raise - shutil.rmtree(backup_path, ignore_errors=True) - shutil.rmtree(_staging_root(installed_dir) / name, ignore_errors=True) - - info = InstallationInfo.from_extension( - manifest, - source=current_info.source, - install_path=install_path, - requested_ref=current_info.requested_ref, - resolved_ref=resolved_ref, - repo_path=current_info.repo_path, + # A directory of its own per apply: the previous bundle is deleted after + # the lock is released, so a shared backup path could be cleared while + # another apply is between its renames. + backup_root = Path( + tempfile.mkdtemp(prefix=f"{name}.previous-", dir=_staging_root(installed_dir)) ) - info.enabled = enabled - - with manager.metadata_session as session: - session.extensions[name] = info + backup_path = backup_root / name + try: + # The swap and the metadata write share one session, so a concurrent + # list can't see the install path missing between the two renames + # and prune the entry. + with manager.metadata_session() as session: + current_info = session.extensions.get(name) + if current_info is None or not install_path.is_dir(): + return None + + # POSIX rename() can't atomically replace a non-empty directory, so + # the active bundle moves aside first; roll back if the second + # rename fails. + os.replace(install_path, backup_path) + try: + os.replace(staged_path, install_path) + except OSError: + os.replace(backup_path, install_path) + raise + + info = InstallationInfo.from_extension( + manifest, + source=current_info.source, + install_path=install_path, + requested_ref=current_info.requested_ref, + resolved_ref=resolved_ref, + repo_path=current_info.repo_path, + ) + info.enabled = enabled + session.extensions[name] = info + finally: + if backup_path.exists() and not install_path.exists(): + logger.error( + f"Could not restore {install_path} after a failed update;" + f" the previous version was kept at {backup_path}" + ) + else: + shutil.rmtree(backup_root, ignore_errors=True) + shutil.rmtree(_staging_root(installed_dir) / name, ignore_errors=True) return info diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py b/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py index 41b289e1315..819ba66b29c 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py @@ -40,7 +40,13 @@ ) from openhands.agent_server.errors import error_responses from openhands.agent_server.server_context import server_context_of -from openhands.sdk.errors import Conflict, Invalid, NotFound, Unprocessable +from openhands.sdk.errors import ( + Conflict, + Invalid, + LockTimeout, + NotFound, + Unprocessable, +) from openhands.sdk.extensions.fetch import ExtensionFetchError from openhands.sdk.utils.redact import redact_url_credentials @@ -203,6 +209,7 @@ class UninstallCanvasExtensionResponse(BaseModel): Invalid: "Could not read canvas extension source", Conflict: "Canvas extension already installed (use force=true)", Unprocessable: "Invalid canvas extension (bad manifest, etc.)", + LockTimeout: "Another extension operation is in progress", } ), ) @@ -251,6 +258,9 @@ async def install_canvas_extension_endpoint( "Source plus Path must point at the extension's own " "directory, not the repository root." ) + except LockTimeout: + # A LockTimeout is also an OSError; it means busy, not invalid. + raise except (ValidationError, ValueError, OSError): raise Unprocessable( "Invalid canvas extension. Ensure the manifest is well-formed." @@ -258,7 +268,11 @@ async def install_canvas_extension_endpoint( @canvas_extensions_router.get( - "/installed", response_model=InstalledCanvasExtensionsListResponse + "/installed", + response_model=InstalledCanvasExtensionsListResponse, + responses=error_responses( + {LockTimeout: "Another extension operation is in progress"} + ), ) def list_installed_canvas_extensions_endpoint() -> ( InstalledCanvasExtensionsListResponse @@ -295,7 +309,12 @@ def get_installed_canvas_extension_endpoint( @canvas_extensions_router.patch( "/installed/{extension_name}", response_model=UpdateCanvasExtensionStateResponse, - responses=error_responses({NotFound: "Canvas extension not installed"}), + responses=error_responses( + { + NotFound: "Canvas extension not installed", + LockTimeout: "Another extension operation is in progress", + } + ), ) async def set_canvas_extension_enabled_endpoint( extension_name: CanvasExtensionNamePath, @@ -308,7 +327,9 @@ async def set_canvas_extension_enabled_endpoint( async with lock: if not request.enabled: await _revoke_and_stop_backend(http_request, extension_name) - if not disable_canvas_extension(name=extension_name): + if not await asyncio.to_thread( + disable_canvas_extension, name=extension_name + ): raise NotFound("Canvas App is not installed") return UpdateCanvasExtensionStateResponse( name=extension_name, enabled=False @@ -356,13 +377,13 @@ async def set_canvas_extension_enabled_endpoint( ) except (ValueError, RuntimeError, OSError) as exc: await _revoke_and_stop_backend(http_request, extension_name) - disable_canvas_extension(name=extension_name) + await asyncio.to_thread(disable_canvas_extension, name=extension_name) raise Conflict(str(exc)) from exc except BaseException: await _revoke_and_stop_backend(http_request, extension_name) - disable_canvas_extension(name=extension_name) + await asyncio.to_thread(disable_canvas_extension, name=extension_name) raise - if not enable_canvas_extension(name=extension_name): + if not await asyncio.to_thread(enable_canvas_extension, name=extension_name): raise NotFound("Canvas App is not installed") return UpdateCanvasExtensionStateResponse(name=extension_name, enabled=True) @@ -370,7 +391,12 @@ async def set_canvas_extension_enabled_endpoint( @canvas_extensions_router.delete( "/installed/{extension_name}", response_model=UninstallCanvasExtensionResponse, - responses=error_responses({NotFound: "Canvas extension not installed"}), + responses=error_responses( + { + NotFound: "Canvas extension not installed", + LockTimeout: "Another extension operation is in progress", + } + ), ) async def uninstall_canvas_extension_endpoint( extension_name: CanvasExtensionNamePath, @@ -378,7 +404,7 @@ async def uninstall_canvas_extension_endpoint( ) -> UninstallCanvasExtensionResponse: """Uninstall a canvas extension by name while retaining backend data.""" await _revoke_and_stop_backend(request, extension_name) - if not uninstall_canvas_extension(name=extension_name): + if not await asyncio.to_thread(uninstall_canvas_extension, name=extension_name): raise NotFound(f"Canvas extension '{extension_name}' is not installed") return UninstallCanvasExtensionResponse( message=f"Canvas extension '{extension_name}' uninstalled" @@ -553,6 +579,9 @@ async def grant_canvas_extension_backend_secrets_endpoint( @canvas_extensions_router.delete( "/installed/{extension_name}/backend/secret-grant", response_model=BackendStatus, + responses=error_responses( + {LockTimeout: "Another extension operation is in progress"} + ), ) async def revoke_canvas_extension_backend_secrets_endpoint( extension_name: CanvasExtensionNamePath, @@ -560,6 +589,6 @@ async def revoke_canvas_extension_backend_secrets_endpoint( ) -> BackendStatus: """Revoke consent, terminate the process group and invalidate bridge sessions.""" async with _lifecycle_lock(request, extension_name): - disable_canvas_extension(name=extension_name) + await asyncio.to_thread(disable_canvas_extension, name=extension_name) await _backend_manager(request).revoke_secrets(extension_name) return await _revoke_and_stop_backend(request, extension_name) diff --git a/openhands-agent-server/openhands/agent_server/plugins_router.py b/openhands-agent-server/openhands/agent_server/plugins_router.py index 5dbd096b445..09b4a21d67d 100644 --- a/openhands-agent-server/openhands/agent_server/plugins_router.py +++ b/openhands-agent-server/openhands/agent_server/plugins_router.py @@ -34,6 +34,7 @@ from openhands.sdk.errors import ( Conflict, Invalid, + LockTimeout, NotFound, Unprocessable, UpstreamFailed, @@ -329,6 +330,7 @@ def fetch_plugin_endpoint(request: FetchPluginRequest) -> FetchPluginResponse: Invalid: "Failed to fetch plugin source", Conflict: "Plugin already installed (use force=true)", Unprocessable: "Invalid plugin (bad name, etc.)", + LockTimeout: "Another extension operation is in progress", } ), ) @@ -350,7 +352,13 @@ def install_plugin_endpoint(request: InstallPluginRequest) -> InstalledPluginRes raise Unprocessable("Invalid plugin. Ensure it has a valid kebab-case name.") -@plugins_router.get("/installed", response_model=InstalledPluginsListResponse) +@plugins_router.get( + "/installed", + response_model=InstalledPluginsListResponse, + responses=error_responses( + {LockTimeout: "Another extension operation is in progress"} + ), +) def list_installed_plugins_endpoint() -> InstalledPluginsListResponse: """List all installed plugins (enabled and disabled).""" plugins = service_list_installed_plugins() @@ -377,7 +385,12 @@ def get_installed_plugin_endpoint( @plugins_router.patch( "/installed/{plugin_name}", response_model=UpdatePluginStateResponse, - responses=error_responses({NotFound: "Plugin not installed"}), + responses=error_responses( + { + NotFound: "Plugin not installed", + LockTimeout: "Another extension operation is in progress", + } + ), ) def set_plugin_enabled_endpoint( plugin_name: PluginNamePath, request: UpdatePluginStateRequest @@ -392,7 +405,12 @@ def set_plugin_enabled_endpoint( @plugins_router.delete( "/installed/{plugin_name}", response_model=UninstallPluginResponse, - responses=error_responses({NotFound: "Plugin not installed"}), + responses=error_responses( + { + NotFound: "Plugin not installed", + LockTimeout: "Another extension operation is in progress", + } + ), ) def uninstall_plugin_endpoint(plugin_name: PluginNamePath) -> UninstallPluginResponse: """Uninstall a plugin by name.""" @@ -408,6 +426,7 @@ def uninstall_plugin_endpoint(plugin_name: PluginNamePath) -> UninstallPluginRes { Invalid: "Failed to fetch the plugin's recorded source", NotFound: "Plugin not installed", + LockTimeout: "Another extension operation is in progress", } ), ) @@ -436,7 +455,13 @@ def refresh_plugin_endpoint(plugin_name: PluginNamePath) -> UpdatePluginResponse ) -@plugins_router.get("/marketplace", response_model=MarketplaceCatalogResponse) +@plugins_router.get( + "/marketplace", + response_model=MarketplaceCatalogResponse, + responses=error_responses( + {LockTimeout: "Another extension operation is in progress"} + ), +) def get_marketplace_catalog() -> MarketplaceCatalogResponse: """Get the plugins marketplace catalog with installation status. diff --git a/openhands-agent-server/openhands/agent_server/skills_router.py b/openhands-agent-server/openhands/agent_server/skills_router.py index 75fa5311003..fd3a957e780 100644 --- a/openhands-agent-server/openhands/agent_server/skills_router.py +++ b/openhands-agent-server/openhands/agent_server/skills_router.py @@ -28,7 +28,13 @@ service_update_skill, sync_public_skills, ) -from openhands.sdk.errors import Conflict, Invalid, NotFound, Unprocessable +from openhands.sdk.errors import ( + Conflict, + Invalid, + LockTimeout, + NotFound, + Unprocessable, +) from openhands.sdk.extensions.fetch import ExtensionFetchError from openhands.sdk.marketplace.registration import MarketplaceRegistration from openhands.sdk.skills import ( @@ -384,6 +390,7 @@ def sync_skills() -> SyncResponse: Invalid: "Failed to fetch skill source", Conflict: "Skill already installed (use force=true)", Unprocessable: "Invalid skill (missing SKILL.md, etc.)", + LockTimeout: "Another extension operation is in progress", } ), ) @@ -422,7 +429,13 @@ def install_skill_endpoint(request: InstallSkillRequest) -> InstalledSkillRespon ) -@skills_router.get("/installed", response_model=InstalledSkillsListResponse) +@skills_router.get( + "/installed", + response_model=InstalledSkillsListResponse, + responses=error_responses( + {LockTimeout: "Another extension operation is in progress"} + ), +) def list_installed_skills_endpoint() -> InstalledSkillsListResponse: """List all installed skills. @@ -464,7 +477,12 @@ def get_installed_skill_endpoint(skill_name: SkillNamePath) -> InstalledSkillRes @skills_router.patch( "/installed/{skill_name}", response_model=UpdateSkillStateResponse, - responses=error_responses({NotFound: "Skill not installed"}), + responses=error_responses( + { + NotFound: "Skill not installed", + LockTimeout: "Another extension operation is in progress", + } + ), ) def set_skill_enabled_endpoint( skill_name: SkillNamePath, request: UpdateSkillStateRequest @@ -494,7 +512,12 @@ def set_skill_enabled_endpoint( @skills_router.delete( "/installed/{skill_name}", response_model=UninstallSkillResponse, - responses=error_responses({NotFound: "Skill not installed"}), + responses=error_responses( + { + NotFound: "Skill not installed", + LockTimeout: "Another extension operation is in progress", + } + ), ) def uninstall_skill_endpoint(skill_name: SkillNamePath) -> UninstallSkillResponse: """Uninstall a skill by name. @@ -525,6 +548,7 @@ def uninstall_skill_endpoint(skill_name: SkillNamePath) -> UninstallSkillRespons { Invalid: "Failed to fetch the skill's recorded source", NotFound: "Skill not installed", + LockTimeout: "Another extension operation is in progress", } ), ) @@ -565,7 +589,13 @@ def refresh_skill_endpoint(skill_name: SkillNamePath) -> UpdateSkillResponse: ) -@skills_router.get("/marketplace", response_model=MarketplaceCatalogResponse) +@skills_router.get( + "/marketplace", + response_model=MarketplaceCatalogResponse, + responses=error_responses( + {LockTimeout: "Another extension operation is in progress"} + ), +) def get_marketplace_catalog() -> MarketplaceCatalogResponse: """Get the marketplace catalog with installation status. diff --git a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py index 68403c5b198..c7d5f2a3839 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py @@ -3,9 +3,14 @@ ExtensionProtocol, InstallationInterface, ) -from openhands.sdk.extensions.installation.manager import InstallationManager +from openhands.sdk.extensions.installation.manager import ( + InstallationManager, + InstallationOutcome, +) from openhands.sdk.extensions.installation.metadata import ( InstallationMetadata, + InstallationMetadataLoadError, + InstallationMetadataLockTimeoutError, MetadataSession, ) @@ -16,5 +21,8 @@ "ExtensionProtocol", "InstallationManager", "InstallationMetadata", + "InstallationMetadataLoadError", + "InstallationMetadataLockTimeoutError", + "InstallationOutcome", "MetadataSession", ] diff --git a/openhands-sdk/openhands/sdk/extensions/installation/manager.py b/openhands-sdk/openhands/sdk/extensions/installation/manager.py index 8e196c75ab4..e5ed6cb18f1 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/manager.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/manager.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import shutil import tempfile from dataclasses import dataclass @@ -31,6 +32,35 @@ DEFAULT_CACHE_DIR = get_user_persistence_dir() / "cache" / "extensions" +@dataclass(frozen=True) +class InstallationOutcome: + """Result of ``InstallationManager.install_with_outcome``.""" + + info: InstallationInfo + was_installed: bool + """Whether an extension of this name was installed (tracked, with its + directory present) when this install replaced it. Decided inside the + metadata session that records the install.""" + + +def _swap_directory(staged_path: Path, install_path: Path, backup_path: Path) -> None: + """Move ``staged_path`` to ``install_path``. + + POSIX ``rename`` can't replace a non-empty directory, so an existing + ``install_path`` is first moved to ``backup_path``. If the second rename + fails, the first is undone and the error is re-raised. + """ + replacing = install_path.exists() + if replacing: + os.replace(install_path, backup_path) + try: + os.replace(staged_path, install_path) + except OSError: + if replacing: + os.replace(backup_path, install_path) + raise + + @dataclass class InstallationManager[T: ExtensionProtocol]: """Generic manager for installing, tracking, and loading extensions. @@ -51,9 +81,11 @@ class InstallationManager[T: ExtensionProtocol]: def __post_init__(self) -> None: self.installation_dir = self.installation_dir.resolve() - @property def metadata_session(self) -> MetadataSession: - """Open a metadata session bound to this manager's dir and interface.""" + """Open a metadata session bound to this manager's dir and interface. + + Takes the metadata lock when called; use it in a ``with`` statement. + """ return InstallationMetadata.open( self.installation_dir, interface=self.installation_interface ) @@ -88,6 +120,22 @@ def install( FileExistsError: If extension is already installed and force=False. ValueError: If the extension name is invalid. """ + return self._install(source, ref=ref, repo_path=repo_path, force=force).info + + def install_with_outcome( + self, + source: str | Path, + ref: str | None = None, + repo_path: str | None = None, + force: bool = False, + ) -> InstallationOutcome: + """Install as ``install`` does, and report whether it replaced an + installed extension. + + For callers whose handling of the result depends on that. Checking + with ``get()`` before calling ``install`` can be out of date by the + time the install is recorded; this is decided while it is recorded. + """ return self._install(source, ref=ref, repo_path=repo_path, force=force) def _install( @@ -97,7 +145,7 @@ def _install( repo_path: str | None = None, force: bool = False, root_name: str | None = None, - ) -> InstallationInfo: + ) -> InstallationOutcome: """Install as ``install`` does; ``root_name`` names a repository root. ``root_name`` replaces the repository's name as the directory name a @@ -127,14 +175,6 @@ def _install( f" at {install_path}. Use force=True to overwrite." ) - if install_path.exists(): - logger.info(f"Removing existing installation of '{extension.name}'") - shutil.rmtree(install_path) - - logger.info(f"Installing extension '{extension.name}' to {install_path}") - self.installation_dir.mkdir(parents=True, exist_ok=True) - shutil.copytree(fetched_path, install_path) - info = InstallationInfo.from_extension( extension, source=source, @@ -144,16 +184,49 @@ def _install( repo_path=repo_path, ) - with self.metadata_session as session: - existing = session.extensions.get(extension.name) - if existing is not None: - info.enabled = existing.enabled - session.extensions[extension.name] = info + # Copy outside the lock into a dot-prefixed directory, which discovery + # skips: a concurrent list never sees a partly copied extension, and + # lock time doesn't grow with its size. Only the swap into place and + # the metadata write happen under the lock. Staging inside the + # installation directory keeps the swap a same-filesystem rename. + logger.info(f"Installing extension '{extension.name}' to {install_path}") + self.installation_dir.mkdir(parents=True, exist_ok=True) + staging_dir = Path( + tempfile.mkdtemp( + prefix=f".{extension.name}.installing-", dir=self.installation_dir + ) + ) + backup_path = staging_dir / "previous" + try: + staged_path = staging_dir / "new" + shutil.copytree(fetched_path, staged_path) + + with self.metadata_session() as session: + if install_path.exists() and not force: + raise FileExistsError( + f"Extension '{extension.name}' is already installed" + f" at {install_path}. Use force=True to overwrite." + ) + existing = session.extensions.get(extension.name) + was_installed = existing is not None and install_path.exists() + if existing is not None: + info.enabled = existing.enabled + _swap_directory(staged_path, install_path, backup_path) + session.extensions[extension.name] = info + finally: + # The previous version is deleted after the lock is released. + if backup_path.exists() and not install_path.exists(): + logger.error( + f"Could not restore {install_path} after a failed install;" + f" the previous version was kept at {backup_path}" + ) + else: + shutil.rmtree(staging_dir, ignore_errors=True) logger.info( f"Successfully installed extension '{extension.name}' v{info.version}" ) - return info + return InstallationOutcome(info=info, was_installed=was_installed) def _load_fetched( self, @@ -202,7 +275,7 @@ def uninstall(self, name: str) -> bool: """ validate_extension_name(name) - with self.metadata_session as session: + with self.metadata_session() as session: if name not in session.extensions: logger.warning(f"Extension '{name}' is not installed") return False @@ -240,7 +313,7 @@ def _set_enabled( ) return False - with self.metadata_session as session: + with self.metadata_session() as session: session.sync() info = session.extensions.get(name) @@ -286,11 +359,7 @@ def list_installed(self, *, strict: bool = False) -> list[InstallationInfo]: if not self.installation_dir.exists(): return [] - with InstallationMetadata.open( - self.installation_dir, - interface=self.installation_interface, - strict=strict, - ) as session: + with self.metadata_session() as session: return session.sync(strict=strict) def load_installed(self, *, strict: bool = False) -> list[T]: @@ -394,4 +463,4 @@ def update(self, name: str) -> InstallationInfo | None: repo_path=current_info.repo_path, force=True, root_name=name, - ) + ).info diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 03a110a9169..af9df5bc623 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -1,21 +1,103 @@ from __future__ import annotations +import os +import stat +import sys from pathlib import Path from types import TracebackType -from typing import Any, ClassVar +from typing import Any, ClassVar, Final +from filelock import BaseFileLock, FileLock, Timeout from pydantic import BaseModel, Field, model_validator +from openhands.sdk.errors import LockTimeout from openhands.sdk.extensions.installation.info import InstallationInfo from openhands.sdk.extensions.installation.interface import ( InstallationInterface, ) from openhands.sdk.extensions.installation.utils import validate_extension_name from openhands.sdk.logger import get_logger +from openhands.sdk.utils.files import atomic_write_text, read_text_retrying logger = get_logger(__name__) +_LOCK_TIMEOUT_SECONDS: Final[float] = 30.0 + + +def _running_as_root() -> bool: + return sys.platform != "win32" and os.geteuid() == 0 + + +def _give_lock_file_to_directory_owner(lock_path: Path) -> None: + """Make the installation directory's owner own the lock file root holds. + + A root process (an agent-server container, say) sharing the directory with + its user would otherwise leave a root-owned lock file that user can never + open again. The file is changed through a descriptor opened without + following symlinks, and only if it is a regular file with a single link, + so the directory's owner can't redirect the change to another file. Best + effort: a failure is logged and the session goes on. + """ + if sys.platform == "win32": + return + try: + directory = lock_path.parent.stat() + fd = os.open(lock_path, os.O_RDONLY | os.O_NOFOLLOW) + try: + current = os.fstat(fd) + if ( + stat.S_ISREG(current.st_mode) + and current.st_nlink == 1 + and (current.st_uid, current.st_gid) + != (directory.st_uid, directory.st_gid) + ): + os.fchown(fd, directory.st_uid, directory.st_gid) + finally: + os.close(fd) + except OSError as e: + logger.warning(f"Could not give {lock_path} to its directory's owner: {e}") + + +class InstallationMetadataLoadError(Exception): + """An existing metadata file could not be read or parsed. + + A write session raises this instead of starting from empty metadata, + because saving that fallback would overwrite every recorded source and + enabled state. The file is left unchanged. + """ + + def __init__(self, metadata_path: Path, cause: BaseException) -> None: + super().__init__( + f"Could not load installation metadata at {metadata_path} " + f"({type(cause).__name__}); the file was left unchanged" + ) + self.metadata_path = metadata_path + + +class InstallationMetadataLockTimeoutError(LockTimeout): + """The installation metadata lock was not acquired in time. + + Another operation (an install, update, uninstall or listing) held it for + the whole timeout. Retrying later can succeed, so the agent server answers + 503. The client sees a generic message; the local path is only logged. + + Like every ``LockTimeout`` it is also a ``TimeoutError``, and so an + ``OSError``: code that maps ``OSError`` to another error must re-raise + ``LockTimeout`` first. + """ + + def __init__(self, installed_dir: Path, timeout: float) -> None: + super().__init__( + "Another plugin, skill or extension operation is in progress. " + "Please retry.", + message=( + f"Timed out after {timeout}s waiting for the installation " + f"metadata lock in {installed_dir}" + ), + ) + self.installed_dir = installed_dir + class MetadataSession: """Context manager that binds ``InstallationMetadata`` to its directory. @@ -32,10 +114,12 @@ def __init__( installed_dir: Path, metadata: InstallationMetadata, interface: InstallationInterface | None = None, + lock: BaseFileLock | None = None, ) -> None: self.installed_dir = installed_dir self.metadata = metadata self.interface = interface + self.lock = lock @property def extensions(self) -> dict[str, InstallationInfo]: @@ -73,8 +157,12 @@ def __exit__( exc_val: BaseException | None, exc_tb: TracebackType | None, ) -> None: - if exc_type is None: - self.metadata.save_to_dir(self.installed_dir) + try: + if exc_type is None: + self.metadata.save_to_dir(self.installed_dir) + finally: + if self.lock is not None: + self.lock.release() class InstallationMetadata(BaseModel): @@ -125,19 +213,56 @@ def open( installed_dir: Path, *, interface: InstallationInterface | None = None, - strict: bool = False, ) -> MetadataSession: """Load metadata and return a session that auto-saves on exit. + Acquires a file lock on ``installed_dir`` before loading and holds it + until the session exits, so a load-modify-save cycle cannot interleave + with another one (another thread or process). Always use the result in + a ``with`` block, and do not nest sessions on the same directory. + Args: installed_dir: Root directory where extensions are installed. interface: Optional installation interface, required if the session will call ``sync()``. - strict: Raise on invalid metadata instead of treating it as empty. + + Raises: + InstallationMetadataLockTimeoutError: Another session held the + lock for the whole timeout. + InstallationMetadataLoadError: The metadata file exists but cannot + be read or parsed. The lock is released and the file is left + unchanged. """ - return MetadataSession( - installed_dir, cls.load_from_dir(installed_dir, strict=strict), interface - ) + installed_dir.mkdir(parents=True, exist_ok=True) + lock = cls._acquire_lock(installed_dir) + try: + return MetadataSession( + installed_dir, + cls._load_for_session(installed_dir), + interface, + lock=lock, + ) + except BaseException: + lock.release() + raise + + @staticmethod + def _acquire_lock(installed_dir: Path) -> BaseFileLock: + lock_path = installed_dir / ".metadata.lock" + lock = FileLock(lock_path) + try: + lock.acquire(timeout=_LOCK_TIMEOUT_SECONDS) + except Timeout as e: + logger.error( + f"Failed to acquire installation metadata lock in " + f"{_LOCK_TIMEOUT_SECONDS}s: {installed_dir}" + ) + raise InstallationMetadataLockTimeoutError( + installed_dir, _LOCK_TIMEOUT_SECONDS + ) from e + if _running_as_root(): + _give_lock_file_to_directory_owner(lock_path) + return lock @classmethod def get_metadata_path(cls, installed_dir: Path) -> Path: @@ -148,24 +273,67 @@ def get_metadata_path(cls, installed_dir: Path) -> Path: def load_from_dir( cls, installed_dir: Path, *, strict: bool = False ) -> InstallationMetadata: - """Load metadata from the installed extensions directory.""" + """Load metadata from the installed extensions directory. + + For read-only callers. Takes no lock: a save swaps in a complete file, + so a concurrent read sees the old or the new version, never a partial + one. On Windows, a read that meets the swap is retried. A file that + cannot be read or parsed loads as empty, or raises with ``strict``. + """ metadata_path = cls.get_metadata_path(installed_dir) if not metadata_path.exists(): return cls() try: - return cls.model_validate_json(metadata_path.read_text(encoding="utf-8")) + return cls.model_validate_json(read_text_retrying(metadata_path)) except Exception as e: if strict: raise logger.warning(f"Failed to load installed extension metadata: {e}") return cls() + @classmethod + def _load_for_session(cls, installed_dir: Path) -> InstallationMetadata: + """Load metadata for a write session. + + Unlike ``load_from_dir``, a file that exists but cannot be read or + parsed raises instead of loading as empty, since the session would + save that empty fallback over it. A missing file still loads as empty. + """ + metadata_path = cls.get_metadata_path(installed_dir) + try: + text = metadata_path.read_text(encoding="utf-8") + except FileNotFoundError: + return cls() + except (OSError, UnicodeDecodeError) as e: + raise InstallationMetadataLoadError(metadata_path, e) from e + try: + return cls.model_validate_json(text) + except Exception as e: + raise InstallationMetadataLoadError(metadata_path, e) from e + def save_to_dir(self, installed_dir: Path) -> None: - """Save metadata to the installed extensions directory.""" + """Save metadata to the installed extensions directory. + + An existing regular file keeps its permission bits, and, when root saves + it, its owner. A new file gets its mode from the umask. + """ metadata_path = self.get_metadata_path(installed_dir) metadata_path.parent.mkdir(parents=True, exist_ok=True) - metadata_path.write_text(self.model_dump_json(indent=2), encoding="utf-8") + mode: int | None = None + owner: tuple[int, int] | None = None + try: + existing = metadata_path.lstat() + except FileNotFoundError: + pass + else: + if stat.S_ISREG(existing.st_mode): + mode = stat.S_IMODE(existing.st_mode) + if _running_as_root(): + owner = (existing.st_uid, existing.st_gid) + atomic_write_text( + metadata_path, self.model_dump_json(indent=2), mode=mode, owner=owner + ) def validate_tracked(self, installed_dir: Path) -> list[InstallationInfo]: """Validate tracked extensions exist on disk. diff --git a/openhands-sdk/openhands/sdk/utils/files.py b/openhands-sdk/openhands/sdk/utils/files.py index cdf2b7e6807..4450b083de7 100644 --- a/openhands-sdk/openhands/sdk/utils/files.py +++ b/openhands-sdk/openhands/sdk/utils/files.py @@ -1,13 +1,33 @@ import os +import secrets +import sys import tempfile +import time +from collections.abc import Callable from pathlib import Path from typing import Final +from openhands.sdk.logger import get_logger + + +logger = get_logger(__name__) # Suffix that marks a file as still being written. ``atomic_write_text`` writes # to ``...tmp`` next to ``path`` and then renames it to ````. _TEMP_SUFFIX: Final[str] = ".tmp" +# On Windows, a file being replaced cannot be opened by anyone else for an +# instant: ``os.replace`` holds the destination open with delete access, and +# Python opens files without delete sharing. So a replace fails while a reader +# has the destination open, and a read fails while a replace is in progress. +# Both raise ``PermissionError``, and both are retried every 10 ms for about +# 1 s before the error is raised. Readers hold the file only for one read, so +# under a steady stream of reads many short attempts get through where a few +# spaced-out ones can each land on a read. On other platforms neither fails +# that way, and neither is retried. +_RETRY_ON_PERMISSION_ERROR: Final[bool] = os.name == "nt" +_PERMISSION_ERROR_RETRY_DELAYS: Final[tuple[float, ...]] = (0.01,) * 100 + def is_temp_file(path: Path) -> bool: """Return True if ``path`` is named like a file that is still being written. @@ -18,25 +38,97 @@ def is_temp_file(path: Path) -> bool: return path.suffix == _TEMP_SUFFIX -def atomic_write_text(path: Path, value: str, mode: int = 0o600) -> None: - """Atomically write text with owner-only permissions.""" - fd, temporary_name = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=_TEMP_SUFFIX, dir=path.parent - ) +def _retry_on_permission_error[T](operation: Callable[[], T]) -> T: + """Run ``operation``, retried while a concurrent replace or read blocks it.""" + if _RETRY_ON_PERMISSION_ERROR: + for delay in _PERMISSION_ERROR_RETRY_DELAYS: + try: + return operation() + except PermissionError: + time.sleep(delay) + return operation() + + +def read_text_retrying(path: Path) -> str: + """Read ``path`` as UTF-8, retried while a replace is in progress. + + For reading, without a lock, a file that ``atomic_write_text`` may replace + at any moment. + """ + return _retry_on_permission_error(lambda: path.read_text(encoding="utf-8")) + + +# ``tempfile.mkstemp``'s flags, for a temp file it can't create: one whose mode +# the umask decides. Windows has no ``O_NOFOLLOW`` and needs ``O_BINARY`` to +# keep the C runtime from translating newlines a second time. +_TEMP_OPEN_FLAGS: Final[int] = ( + os.O_RDWR + | os.O_CREAT + | os.O_EXCL + | (os.O_BINARY if sys.platform == "win32" else os.O_NOFOLLOW) +) + + +def _create_temp_file_with_umask_mode(path: Path) -> tuple[int, str]: + """Create a temp file next to ``path`` the way ``open()`` creates a file. + + Like ``tempfile.mkstemp``, but the file starts as ``0o666`` and the umask + removes bits from it, where ``mkstemp`` always creates ``0o600``. The umask + itself can only be read by setting it, which would change it for every + thread in the process for that moment. + """ + for _ in range(100): + name = os.path.join( + path.parent, f".{path.name}.{secrets.token_hex(8)}{_TEMP_SUFFIX}" + ) + try: + return os.open(name, _TEMP_OPEN_FLAGS, 0o666), name + except FileExistsError: + continue + raise FileExistsError(f"No unused temporary file name next to {path}") + + +def atomic_write_text( + path: Path, + value: str, + mode: int | None = 0o600, + owner: tuple[int, int] | None = None, +) -> None: + """Atomically write text to ``path`` by replacing it. + + ``mode`` is the permission bits ``path`` ends up with: owner-only by + default. ``None`` leaves them to the umask, as for a file ``open()`` creates. + + ``owner`` is a ``(uid, gid)`` for ``path`` to end up with, on POSIX. It is + best effort: if it can't be applied, ``path`` keeps the writer's owner and + a warning is logged. + """ + if mode is None: + fd, temporary_name = _create_temp_file_with_umask_mode(path) + else: + fd, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=_TEMP_SUFFIX, dir=path.parent + ) temporary_path = Path(temporary_name) try: - fchmod = getattr(os, "fchmod", None) - if fchmod is None: - os.chmod(temporary_path, mode) - else: - fchmod(fd, mode) + if mode is not None: + fchmod = getattr(os, "fchmod", None) + if fchmod is None: + os.chmod(temporary_path, mode) + else: + fchmod(fd, mode) + if owner is not None and sys.platform != "win32": + try: + os.fchown(fd, *owner) + except OSError as e: + logger.warning(f"Could not keep the owner of {path}: {e}") file = os.fdopen(fd, "w", encoding="utf-8") fd = -1 with file: file.write(value) file.flush() os.fsync(file.fileno()) - os.replace(temporary_path, path) + _retry_on_permission_error(lambda: os.replace(temporary_path, path)) except BaseException: if fd >= 0: try: diff --git a/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py b/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py index 4703528e9ac..12ee1818a91 100644 --- a/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py +++ b/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py @@ -8,6 +8,9 @@ import inspect import json +import os +import shutil +import threading from pathlib import Path from typing import Any @@ -15,6 +18,7 @@ from pydantic import ValidationError from openhands.agent_server.canvas_extensions.installed import ( + InstalledCanvasExtensionInfo, disable_canvas_extension, enable_canvas_extension, get_installed_canvas_extension, @@ -25,7 +29,10 @@ uninstall_canvas_extension, ) from openhands.agent_server.canvas_extensions.manifest import MANIFEST_FILENAME -from openhands.sdk.extensions.installation import InstallationMetadata +from openhands.sdk.extensions.installation import ( + InstallationMetadata, + manager as installation_manager, +) from .conftest import write_extension as _write_extension @@ -397,3 +404,133 @@ def test_disable_nonexistent_extension_returns_false(installed_dir: Path): def test_enable_nonexistent_extension_returns_false(installed_dir: Path): assert enable_canvas_extension("nonexistent", installed_dir=installed_dir) is False + + +# ============================================================================ +# Force reinstall racing other operations +# ============================================================================ + + +def test_force_reinstall_with_a_concurrent_list_keeps_disabled_state( + extension_dir: Path, installed_dir: Path, monkeypatch: pytest.MonkeyPatch +): + """A list call while a force reinstall copies the new bundle must not + prune the disabled entry: the reinstall would then record it as new and + enabled, skipping the approval that enabling requires. + """ + install_canvas_extension(str(extension_dir), installed_dir=installed_dir) + real_copytree = shutil.copytree + listed: list[list[Any]] = [] + + def copytree(src, dst, *args, **kwargs): + if Path(dst).is_relative_to(installed_dir.resolve()) and not listed: + thread = threading.Thread( + target=lambda: listed.append( + list_installed_canvas_extensions(installed_dir=installed_dir) + ) + ) + thread.start() + thread.join(10) + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(installation_manager.shutil, "copytree", copytree) + + info = install_canvas_extension( + str(extension_dir), installed_dir=installed_dir, force=True + ) + + assert [(i.name, i.enabled) for i in listed[0]] == [("my-extension", False)] + assert info.enabled is False + on_disk = get_installed_canvas_extension( + "my-extension", installed_dir=installed_dir + ) + assert on_disk is not None + assert on_disk.enabled is False + + +def test_force_reinstall_after_a_concurrent_uninstall_lands_disabled( + extension_dir: Path, installed_dir: Path, monkeypatch: pytest.MonkeyPatch +): + """Whether a force install replaced an installed extension is decided when + the install is recorded. Decided earlier, an uninstall in between would + let the new entry pass as a reinstall and land enabled. + """ + install_canvas_extension(str(extension_dir), installed_dir=installed_dir) + enable_canvas_extension("my-extension", installed_dir=installed_dir) + real_copytree = shutil.copytree + uninstalled: list[bool] = [] + + def copytree(src, dst, *args, **kwargs): + if not uninstalled: + uninstalled.append( + uninstall_canvas_extension("my-extension", installed_dir=installed_dir) + ) + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(installation_manager.shutil, "copytree", copytree) + + info = install_canvas_extension( + str(extension_dir), installed_dir=installed_dir, force=True + ) + + assert uninstalled == [True] + assert info.enabled is False + on_disk = InstallationMetadata.load_from_dir(installed_dir) + assert on_disk.extensions["my-extension"].enabled is False + + +def test_list_during_a_reinstall_keeps_an_enabled_extension_enabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """A list landing while a force reinstall swaps the directory in must not + take the installed extension for a new one and disable it. + """ + installed_dir = tmp_path / "installed" + source = _write_extension( + tmp_path / "src" / "demo-extension", name="demo-extension" + ) + install_canvas_extension(source=str(source), installed_dir=installed_dir) + assert enable_canvas_extension("demo-extension", installed_dir=installed_dir) + + list_waiting = threading.Event() + acquire_lock = InstallationMetadata._acquire_lock + + def signalling_acquire_lock(directory: Path): + if threading.current_thread().name == "list": + list_waiting.set() + return acquire_lock(directory) + + monkeypatch.setattr( + InstallationMetadata, "_acquire_lock", staticmethod(signalling_acquire_lock) + ) + real_replace = os.replace + lists: list[threading.Thread] = [] + listed: list[list[InstalledCanvasExtensionInfo]] = [] + + def replace(src, dst) -> None: + real_replace(src, dst) + if Path(src) == installed_dir / "demo-extension" and not lists: + thread = threading.Thread( + target=lambda: listed.append( + list_installed_canvas_extensions(installed_dir=installed_dir) + ), + name="list", + daemon=True, + ) + lists.append(thread) + thread.start() + assert list_waiting.wait(5) + + monkeypatch.setattr(installation_manager.os, "replace", replace) + + install_canvas_extension( + source=str(source), installed_dir=installed_dir, force=True + ) + + assert len(lists) == 1, "the reinstall did not move the installed directory aside" + lists[0].join(10) + assert not lists[0].is_alive(), "the concurrent list did not finish" + assert [(i.name, i.enabled) for i in listed[0]] == [("demo-extension", True)] + info = get_installed_canvas_extension("demo-extension", installed_dir=installed_dir) + assert info is not None + assert info.enabled is True diff --git a/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py b/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py index 19fd8785f6d..4ea58fe833c 100644 --- a/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py +++ b/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py @@ -2,6 +2,7 @@ import json import os +import threading from pathlib import Path from unittest.mock import patch @@ -15,10 +16,12 @@ enable_canvas_extension, get_installed_canvas_extension, install_canvas_extension, + list_installed_canvas_extensions, uninstall_canvas_extension, ) from openhands.agent_server.canvas_extensions.manifest import MANIFEST_FILENAME from openhands.sdk.extensions.fetch import ExtensionFetchError +from openhands.sdk.extensions.installation import InstallationMetadata from .conftest import write_extension as _write_extension @@ -523,7 +526,7 @@ def test_apply_cleans_up_staging_after_success(extension_dir: Path, installed: P ) assert not (installed / ".staging" / "my-extension").exists() - assert not (installed / ".staging" / "my-extension.previous").exists() + assert list((installed / ".staging").glob("my-extension.previous*")) == [] def test_double_apply_without_recheck_raises(extension_dir: Path, installed: Path): @@ -674,6 +677,97 @@ def test_apply_leaves_no_backup_when_first_rename_fails( assert staged.is_dir() +def test_apply_keeps_the_previous_bundle_when_the_rollback_fails( + extension_dir: Path, installed: Path +): + """If the rollback rename fails too, the previous bundle is the only copy + left, so it stays in its backup directory instead of being deleted. + """ + _write_extension(extension_dir, version="2.0.0") + check = check_canvas_extension_update("my-extension", installed_dir=installed) + assert check is not None + install_path = installed / "my-extension" + real_replace = os.replace + renames = {"n": 0} + + def flaky_replace(src, dst): + if install_path in (Path(src), Path(dst)): + renames["n"] += 1 + if renames["n"] in (2, 3): + raise OSError("simulated rename failure") + return real_replace(src, dst) + + with patch("os.replace", side_effect=flaky_replace): + with pytest.raises(OSError, match="simulated rename failure"): + apply_canvas_extension_update( + "my-extension", + resolved_ref=check.resolved_ref, + enabled=True, + installed_dir=installed, + ) + + assert not install_path.exists() + [backup_root] = (installed / ".staging").glob("my-extension.previous-*") + assert _active_version(backup_root) == "1.0.0" + + +def test_list_cannot_run_between_the_apply_renames( + extension_dir: Path, installed: Path, monkeypatch: pytest.MonkeyPatch +): + """The install path is missing between apply's two renames. The metadata + lock must keep a list call from loading in that window, where it would + prune the entry. + """ + _write_extension(extension_dir, version="2.0.0") + check = check_canvas_extension_update("my-extension", installed_dir=installed) + assert check is not None + install_path = installed / "my-extension" + original_load = InstallationMetadata._load_for_session.__func__ + list_loaded = threading.Event() + + def load(cls: type[InstallationMetadata], directory: Path): + result = original_load(cls, directory) + if threading.current_thread().name == "list": + list_loaded.set() + return result + + monkeypatch.setattr(InstallationMetadata, "_load_for_session", classmethod(load)) + real_replace = os.replace + lists: list[threading.Thread] = [] + listed: list[list] = [] + + def replace(src, dst): + real_replace(src, dst) + if Path(src) == install_path and not lists: + thread = threading.Thread( + target=lambda: listed.append( + list_installed_canvas_extensions(installed_dir=installed) + ), + name="list", + daemon=True, + ) + lists.append(thread) + thread.start() + assert not list_loaded.wait(0.5), ( + "list loaded metadata between the apply renames" + ) + + monkeypatch.setattr(os, "replace", replace) + + apply_canvas_extension_update( + "my-extension", + resolved_ref=check.resolved_ref, + enabled=False, + installed_dir=installed, + ) + + assert len(lists) == 1, "apply did not move the active bundle aside" + lists[0].join(10) + assert [(i.name, i.version, i.enabled) for i in listed[0]] == [ + ("my-extension", "2.0.0", False) + ] + + def test_apply_rejects_manifest_that_became_invalid_since_check( extension_dir: Path, installed: Path ): diff --git a/tests/agent_server/test_canvas_extensions_lock_responsiveness.py b/tests/agent_server/test_canvas_extensions_lock_responsiveness.py new file mode 100644 index 00000000000..5337c0598a2 --- /dev/null +++ b/tests/agent_server/test_canvas_extensions_lock_responsiveness.py @@ -0,0 +1,145 @@ +"""A Canvas request waiting on the installation metadata lock must not stall +the server. + +``InstallationMetadata.open()`` takes a ``filelock.FileLock`` whose ``acquire`` +polls with a blocking sleep (up to 30 s). The ``async def`` Canvas endpoints +must run the lock-taking calls off the event loop, as the install endpoint +already does, so ``/alive`` keeps answering while another process holds the +lock. Same check as ``test_profile_store_lock_responsiveness.py``, except that +these handlers stop the backend before taking the lock, so the test waits for +the lock attempt itself instead of a fixed delay. +""" + +import asyncio +import threading +import time +from pathlib import Path + +import httpx +import pytest +from filelock import FileLock + +from openhands.agent_server.api import create_app +from openhands.agent_server.canvas_extensions.installed import ( + enable_canvas_extension, + install_canvas_extension, +) +from openhands.agent_server.config import Config +from openhands.sdk.extensions.installation import InstallationMetadata + +from .canvas_extensions.conftest import write_extension + + +# Upper bound for the held lock: if the handler blocks the event loop, nothing +# can release the lock from the test, so a timer frees it after this long. +_SAFETY_RELEASE_SECONDS = 5.0 +_ALIVE_BUDGET_SECONDS = 2.0 +_NAME = "demo-extension" + + +@pytest.mark.parametrize( + ("method", "path", "json_body", "enabled_before"), + [ + pytest.param( + "PATCH", + f"/api/canvas-extensions/installed/{_NAME}", + {"enabled": False}, + True, + id="disable", + ), + pytest.param( + "PATCH", + f"/api/canvas-extensions/installed/{_NAME}", + {"enabled": True}, + False, + id="enable", + ), + pytest.param( + "DELETE", + f"/api/canvas-extensions/installed/{_NAME}", + None, + False, + id="uninstall", + ), + pytest.param( + "DELETE", + f"/api/canvas-extensions/installed/{_NAME}/backend/secret-grant", + None, + True, + id="revoke-secret-grant", + ), + ], +) +async def test_alive_answers_while_canvas_request_waits_for_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + json_body: dict | None, + enabled_before: bool, +) -> None: + installed_dir = tmp_path / "canvas-installed" + monkeypatch.setattr( + "openhands.agent_server.canvas_extensions.installed." + "get_installed_canvas_extensions_dir", + lambda: installed_dir, + ) + monkeypatch.setenv("OH_PERSISTENCE_DIR", str(tmp_path)) + install_canvas_extension( + source=str(write_extension(tmp_path / "src" / _NAME, name=_NAME)) + ) + if enabled_before: + assert enable_canvas_extension(_NAME) + app = create_app( + Config(static_files_path=None, session_api_keys=[], secret_key=None) + ) + + # Set from the thread that is about to wait on the lock. The time is taken + # there too: if the handler blocks the event loop, the test's own clock + # reading would only resume after the lock is released. + lock_requested = threading.Event() + lock_requested_at: list[float] = [] + acquire_lock = InstallationMetadata._acquire_lock + + def signalling_acquire_lock(installed_dir: Path): + lock_requested_at.append(time.monotonic()) + lock_requested.set() + return acquire_lock(installed_dir) + + monkeypatch.setattr( + InstallationMetadata, "_acquire_lock", staticmethod(signalling_acquire_lock) + ) + + # A separate FileLock instance stands in for another process holding the + # metadata lock. ``thread_local=False`` lets the safety timer release it. + held = FileLock(installed_dir / ".metadata.lock", thread_local=False) + held.acquire() + safety = threading.Timer(_SAFETY_RELEASE_SECONDS, held.release) + safety.start() + try: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + waiting = asyncio.create_task( + client.request(method, path, json=json_body, timeout=60) + ) + assert await asyncio.to_thread(lock_requested.wait, 10) + alive = await client.get("/alive", timeout=60) + elapsed = time.monotonic() - lock_requested_at[0] + + assert alive.status_code == 200 + assert elapsed < _ALIVE_BUDGET_SECONDS, ( + f"/alive took {elapsed:.1f}s while a Canvas request waited " + "for the metadata lock: the wait is blocking the event loop" + ) + # The Canvas request was really still waiting on the lock. + assert not waiting.done() + + safety.cancel() + held.release() + response = await asyncio.wait_for(waiting, timeout=30) + assert response.status_code == 200, response.text + finally: + safety.cancel() + if held.is_locked: + held.release(force=True) diff --git a/tests/agent_server/test_installation_lock_timeout.py b/tests/agent_server/test_installation_lock_timeout.py new file mode 100644 index 00000000000..426d5a5b03c --- /dev/null +++ b/tests/agent_server/test_installation_lock_timeout.py @@ -0,0 +1,141 @@ +"""A plugin, skill or Canvas request that times out on the installation +metadata lock answers 503, not a 500 or a 422. + +Another holder keeps the real lock file locked while each endpoint runs, with +the lock timeout cut to 0.2 s. Goes through ``create_app`` because the mapping +is an app-level exception handler. +""" + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from filelock import FileLock + +from openhands.agent_server.api import create_app +from openhands.agent_server.config import Config +from openhands.sdk.extensions.installation import metadata as metadata_module + +from .canvas_extensions.conftest import write_extension + + +_BUSY_DETAIL = ( + "Another plugin, skill or extension operation is in progress. Please retry." +) + + +@pytest.fixture +def stores(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path]: + plugins = tmp_path / "plugins-installed" + skills = tmp_path / "skills-installed" + canvas = tmp_path / "canvas-installed" + for store in (plugins, skills, canvas): + store.mkdir() + monkeypatch.setattr( + "openhands.sdk.plugin.installed.DEFAULT_INSTALLED_PLUGINS_DIR", plugins + ) + monkeypatch.setattr( + "openhands.sdk.skills.installed.DEFAULT_INSTALLED_SKILLS_DIR", skills + ) + monkeypatch.setattr( + "openhands.agent_server.canvas_extensions.installed." + "get_installed_canvas_extensions_dir", + lambda: canvas, + ) + # The marketplace routes clone their catalog with git before listing. + empty_repo = tmp_path / "empty-marketplace-repo" + empty_repo.mkdir() + for service in ("plugins_service", "skills_service"): + monkeypatch.setattr( + f"openhands.agent_server.{service}.update_skills_repository", + lambda *args, **kwargs: empty_repo, + ) + return {"plugins": plugins, "skills": skills, "canvas": canvas} + + +@pytest.fixture +def client(tmp_path: Path, stores: dict[str, Path]) -> TestClient: + client = TestClient( + create_app(Config(session_api_keys=[])), raise_server_exceptions=False + ) + plugin = tmp_path / "src" / "demo-plugin" + (plugin / ".plugin").mkdir(parents=True) + (plugin / ".plugin" / "plugin.json").write_text( + json.dumps({"name": "demo-plugin", "version": "1.0.0", "description": "x"}) + ) + skill = tmp_path / "src" / "demo-skill" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text( + "---\nname: demo-skill\ndescription: A test skill\n---\nBody\n" + ) + canvas = write_extension(tmp_path / "src" / "demo-canvas", name="demo-canvas") + for route, source in ( + ("/api/plugins/install", plugin), + ("/api/skills/install", skill), + ("/api/canvas-extensions/install", canvas), + ): + response = client.post(route, json={"source": str(source)}) + assert response.status_code == 200, response.text + return client + + +@pytest.mark.parametrize( + ("store", "method", "path", "json_body"), + [ + ("plugins", "POST", "/api/plugins/install", "source"), + ("plugins", "GET", "/api/plugins/installed", None), + ("plugins", "PATCH", "/api/plugins/installed/demo-plugin", {"enabled": False}), + ("plugins", "DELETE", "/api/plugins/installed/demo-plugin", None), + ("plugins", "POST", "/api/plugins/installed/demo-plugin/refresh", None), + ("plugins", "GET", "/api/plugins/marketplace", None), + ("skills", "POST", "/api/skills/install", "source"), + ("skills", "GET", "/api/skills/installed", None), + ("skills", "PATCH", "/api/skills/installed/demo-skill", {"enabled": False}), + ("skills", "DELETE", "/api/skills/installed/demo-skill", None), + ("skills", "POST", "/api/skills/installed/demo-skill/refresh", None), + ("skills", "GET", "/api/skills/marketplace", None), + ("canvas", "POST", "/api/canvas-extensions/install", "source"), + ("canvas", "GET", "/api/canvas-extensions/installed", None), + ( + "canvas", + "PATCH", + "/api/canvas-extensions/installed/demo-canvas", + {"enabled": True}, + ), + ("canvas", "DELETE", "/api/canvas-extensions/installed/demo-canvas", None), + ( + "canvas", + "DELETE", + "/api/canvas-extensions/installed/demo-canvas/backend/secret-grant", + None, + ), + ], +) +def test_lock_timeout_returns_503( + client: TestClient, + stores: dict[str, Path], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + store: str, + method: str, + path: str, + json_body: dict | str | None, +): + if json_body == "source": + # A force reinstall from a fresh source, so the request gets past the + # "already installed" check and fetching, up to the lock. + name = {"plugins": "demo-plugin", "skills": "demo-skill"}.get( + store, "demo-canvas" + ) + json_body = {"source": str(tmp_path / "src" / name), "force": True} + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + + held = FileLock(stores[store] / ".metadata.lock") + with held: + response = client.request(method, path, json=json_body) + + assert response.status_code == 503, response.text + # The client is told to retry and never sees the local path. + assert _BUSY_DETAIL in response.json()["exception"] + assert str(stores[store]) not in response.text diff --git a/tests/agent_server/test_plugins_router.py b/tests/agent_server/test_plugins_router.py index 3b035935839..269d8d65b0b 100644 --- a/tests/agent_server/test_plugins_router.py +++ b/tests/agent_server/test_plugins_router.py @@ -7,6 +7,7 @@ import json import shutil import subprocess +import threading from pathlib import Path from unittest.mock import patch @@ -128,6 +129,43 @@ def test_refresh_returns_updated_plugin(client: TestClient, tmp_path: Path): assert refreshed.json()["plugin"]["name"] == "demo-plugin" +def test_refresh_with_a_concurrent_list_keeps_a_disabled_plugin_disabled( + client: TestClient, tmp_path: Path, monkeypatch +): + """#5985 through the API: a GET /plugins/installed while a refresh copies + the new version must not prune the plugin's entry, which would make the + refresh re-enable a disabled plugin. + """ + src = _make_installable_plugin(tmp_path / "src", "demo-plugin") + client.post("/plugins/install", json={"source": str(src)}) + client.patch("/plugins/installed/demo-plugin", json={"enabled": False}) + store = (tmp_path / "installed-store").resolve() + real_copytree = shutil.copytree + listed = [] + + def copytree(source, destination, *args, **kwargs): + if Path(destination).is_relative_to(store) and not listed: + thread = threading.Thread( + target=lambda: listed.append(client.get("/plugins/installed")) + ) + thread.start() + thread.join(10) + return real_copytree(source, destination, *args, **kwargs) + + monkeypatch.setattr(shutil, "copytree", copytree) + + refreshed = client.post("/plugins/installed/demo-plugin/refresh") + + assert refreshed.status_code == 200 + assert len(listed) == 1, "the refresh did not copy into the install store" + assert listed[0].status_code == 200 + assert [(p["name"], p["enabled"]) for p in listed[0].json()["plugins"]] == [ + ("demo-plugin", False) + ] + plugin = client.get("/plugins/installed/demo-plugin").json() + assert (plugin["enabled"], plugin["source"]) == (False, str(src)) + + def test_refresh_vanished_source_is_400_and_keeps_plugin( client: TestClient, tmp_path: Path ): diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py new file mode 100644 index 00000000000..8d19900c61a --- /dev/null +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -0,0 +1,488 @@ +"""Concurrency regression tests for installation metadata""" + +import json +import os +import shutil +import threading +from collections.abc import Callable +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from openhands.sdk.errors import LockTimeout +from openhands.sdk.extensions.installation import ( + InstallationInfo, + InstallationInterface, + InstallationManager, + InstallationMetadata, + InstallationMetadataLockTimeoutError, + MetadataSession, + manager as manager_module, + metadata as metadata_module, +) + + +# Disable/enable pairs the writer runs while the reader reads. +_SAVES = 100 + + +class MockExtension(BaseModel): + name: str + version: str = "0.0.1" + description: str = "Mock extension" + + +class MockExtensionInstallationInterface(InstallationInterface): + @staticmethod + def load_from_dir(extension_dir: Path) -> MockExtension: + return MockExtension.model_validate_json( + (extension_dir / "extension.json").read_text() + ) + + +@pytest.fixture +def source_dir(tmp_path: Path) -> Path: + directory = tmp_path / "src" / "magic-test" + directory.mkdir(parents=True) + (directory / "extension.json").write_text( + MockExtension(name="magic-test").model_dump_json() + ) + return directory + + +@pytest.fixture +def installation_dir(tmp_path: Path) -> Path: + directory = tmp_path / "installed" + directory.mkdir() + return directory + + +@pytest.fixture +def manager(installation_dir: Path) -> InstallationManager[MockExtension]: + return InstallationManager( + installation_dir=installation_dir, + installation_interface=MockExtensionInstallationInterface(), + ) + + +def _run_threads(*targets: tuple[str, Callable[[], object]]) -> list[BaseException]: + """Run each target in a named daemon thread; return exceptions they raised.""" + errors: list[BaseException] = [] + + def capture(fn: Callable[[], object]) -> Callable[[], None]: + def run() -> None: + try: + fn() + except BaseException as e: + errors.append(e) + + return run + + threads = [ + threading.Thread(target=capture(fn), name=name, daemon=True) + for name, fn in targets + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(10) + assert not thread.is_alive(), f"thread {thread.name!r} did not finish" + return errors + + +@pytest.mark.parametrize("operation", ["install", "disable"]) +def test_list_cannot_load_metadata_while_another_session_is_open( + manager: InstallationManager[MockExtension], + installation_dir: Path, + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, + operation: str, +): + """Lost update: a list call that loads metadata before another session + saves later overwrites that session's change with a stale copy (for a fresh + install, plus a ``source="local"`` rediscovery). The session lock must keep + the list call from loading until the other session has saved. + """ + if operation == "disable": + manager.install(str(source_dir)) + + original_load = InstallationMetadata._load_for_session.__func__ + writer_loading = threading.Event() + list_loaded = threading.Event() + + def load(cls: type[InstallationMetadata], directory: Path): + me = threading.current_thread().name + if me == "writer": + writer_loading.set() + assert not list_loaded.wait(0.5), ( + "list_installed loaded metadata while another session was open" + ) + result = original_load(cls, directory) + if me == "list": + list_loaded.set() + return result + + monkeypatch.setattr(InstallationMetadata, "_load_for_session", classmethod(load)) + + def write() -> None: + if operation == "install": + manager.install(str(source_dir)) + else: + manager.disable("magic-test") + + def list_once_writer_is_loading() -> None: + assert writer_loading.wait(5) + manager.list_installed() + + errors = _run_threads(("writer", write), ("list", list_once_writer_is_loading)) + + assert errors == [] + data = json.loads( + InstallationMetadata.get_metadata_path(installation_dir).read_text() + ) + entry = data["extensions"]["magic-test"] + assert entry["source"] == str(source_dir) + assert entry["enabled"] is (operation == "install") + + +def test_reader_thread_finds_the_extension_during_saves( + manager: InstallationManager[MockExtension], + source_dir: Path, +): + """``get()`` takes no lock, so while another thread saves it must still + find the extension: never a truncated file (which loads as empty). On + Windows this also covers both sides of the replace: a save that meets the + reader's open handle, and a read that meets the save's replace, must each + retry instead of failing. + """ + manager.install(str(source_dir)) + saving = threading.Event() + done = threading.Event() + reads: list[InstallationInfo | None] = [] + + def save_repeatedly() -> None: + try: + for _ in range(_SAVES): + manager.disable("magic-test") + saving.set() + manager.enable("magic-test") + finally: + done.set() + + def read_until_done() -> None: + assert saving.wait(5) + while not done.is_set(): + reads.append(manager.get("magic-test")) + + errors = _run_threads(("writer", save_repeatedly), ("reader", read_until_done)) + + missed = [info for info in reads if info is None or info.source != str(source_dir)] + thread_errors = [f"{type(e).__name__}: {e}" for e in errors] + assert (thread_errors, len(missed)) == ([], 0), ( + f"thread errors: {thread_errors}; " + f"{len(missed)} of {len(reads)} reads did not find the extension" + ) + assert reads, "the reader never ran while the writer was saving" + + +def test_get_does_not_wait_for_an_open_session( + manager: InstallationManager[MockExtension], + installation_dir: Path, + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """``get()`` is read-only and takes no lock, so a session held open by + another caller must not block it or make it time out. + """ + manager.install(str(source_dir)) + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + results: list[InstallationInfo | None] = [] + + with InstallationMetadata.open(installation_dir): + errors = _run_threads( + ("get", lambda: results.append(manager.get("magic-test"))) + ) + + assert errors == [] + assert len(results) == 1 + assert results[0] is not None + assert results[0].source == str(source_dir) + + +def test_session_lock_times_out_instead_of_blocking_forever( + installation_dir: Path, monkeypatch: pytest.MonkeyPatch +): + """A session that is never closed must not hang every other caller.""" + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + + with InstallationMetadata.open(installation_dir): + errors = _run_threads( + ("other", lambda: InstallationMetadata.open(installation_dir)), + ) + + assert len(errors) == 1 + assert isinstance(errors[0], InstallationMetadataLockTimeoutError) + # The agent server maps every LockTimeout to 503. + assert isinstance(errors[0], LockTimeout) + assert errors[0].installed_dir == installation_dir + + +def _raise_runtime_error(*args, **kwargs): + raise RuntimeError("boom") + + +@pytest.mark.parametrize( + "failure", + ["error_inside_session", "save_fails", "load_fails"], +) +def test_lock_is_released_when_a_session_fails( + installation_dir: Path, monkeypatch: pytest.MonkeyPatch, failure: str +): + """A failing session must release the lock deterministically. + + The session and ``excinfo`` are kept referenced, as a caller that logs or + re-raises the exception would keep them. filelock also releases on garbage + collection, so without those references a missing release goes unnoticed. + """ + sessions: list[MetadataSession] = [] + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + + with monkeypatch.context() as patch: + if failure == "save_fails": + patch.setattr(InstallationMetadata, "save_to_dir", _raise_runtime_error) + elif failure == "load_fails": + patch.setattr( + InstallationMetadata, + "_load_for_session", + classmethod(_raise_runtime_error), + ) + with pytest.raises(RuntimeError) as excinfo: + with InstallationMetadata.open(installation_dir) as session: + sessions.append(session) + if failure == "error_inside_session": + raise RuntimeError("boom") + + errors = _run_threads( + ( + "next", + lambda: InstallationMetadata.open(installation_dir).__exit__( + None, None, None + ), + ), + ) + assert errors == [] + assert excinfo.value is not None + assert len(sessions) == (0 if failure == "load_fails" else 1) + + +# ============================================================================ +# Directory swap during install(force=True) and update() +# ============================================================================ + + +def _staging_dirs(manager: InstallationManager[MockExtension]) -> list[Path]: + return list(manager.installation_dir.glob(".magic-test.installing-*")) + + +def _install_disabled_with_new_version( + manager: InstallationManager[MockExtension], source_dir: Path +) -> Path: + """Install and disable magic-test, then add a file only the new version has.""" + manager.install(str(source_dir)) + manager.disable("magic-test") + (source_dir / "new-version").write_text("2") + return manager.installation_dir / "magic-test" + + +def test_list_while_update_copies_keeps_the_entry( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """#5985: a list call while update() copies the new version must not see + the extension missing (and prune its entry) or half copied (and rediscover + it as local), so a disabled extension stays disabled. + """ + _install_disabled_with_new_version(manager, source_dir) + real_copytree = shutil.copytree + listed: list[list[InstallationInfo]] = [] + + def copytree(src, dst, *args, **kwargs): + if Path(dst).is_relative_to(manager.installation_dir) and not listed: + errors = _run_threads( + ("list", lambda: listed.append(manager.list_installed())) + ) + assert errors == [] + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(manager_module.shutil, "copytree", copytree) + + manager.update("magic-test") + + assert len(listed) == 1, "update() did not copy into the installation directory" + assert [(i.name, i.source, i.enabled) for i in listed[0]] == [ + ("magic-test", str(source_dir), False) + ] + info = manager.get("magic-test") + assert info is not None + assert (info.source, info.enabled) == (str(source_dir), False) + assert (manager.installation_dir / "magic-test" / "new-version").exists() + assert _staging_dirs(manager) == [] + + +def test_list_cannot_run_between_the_swap_renames( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """The swap leaves the install path missing between its two renames. The + metadata lock must keep a list call from loading in that window, where it + would prune the entry. + """ + install_path = _install_disabled_with_new_version(manager, source_dir) + original_load = InstallationMetadata._load_for_session.__func__ + list_loaded = threading.Event() + + def load(cls: type[InstallationMetadata], directory: Path): + result = original_load(cls, directory) + if threading.current_thread().name == "list": + list_loaded.set() + return result + + monkeypatch.setattr(InstallationMetadata, "_load_for_session", classmethod(load)) + real_replace = os.replace + lists: list[threading.Thread] = [] + listed: list[list[InstallationInfo]] = [] + + def replace(src, dst) -> None: + real_replace(src, dst) + if Path(src) == install_path and not lists: + thread = threading.Thread( + target=lambda: listed.append(manager.list_installed()), + name="list", + daemon=True, + ) + lists.append(thread) + thread.start() + assert not list_loaded.wait(0.5), ( + "list_installed loaded metadata between the swap renames" + ) + + monkeypatch.setattr(manager_module.os, "replace", replace) + + manager.update("magic-test") + + assert len(lists) == 1, "update() did not move the installed directory aside" + lists[0].join(10) + assert [(i.name, i.enabled) for i in listed[0]] == [("magic-test", False)] + + +@pytest.mark.parametrize( + "failing_renames", + [{1}, {2}, {2, 3}], + ids=["first_rename", "second_rename", "second_rename_and_rollback"], +) +def test_failed_swap_keeps_the_installed_version( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, + failing_renames: set[int], +): + """A rename that fails during the swap (on Windows: a file in the + directory is open) must leave the previous version installed and its + metadata unchanged. If even the rollback fails, the previous version is + kept in the staging directory rather than deleted. + """ + install_path = _install_disabled_with_new_version(manager, source_dir) + real_replace = os.replace + renames: list[tuple[Path, Path]] = [] + + def replace(src, dst) -> None: + if install_path in (Path(src), Path(dst)): + renames.append((Path(src), Path(dst))) + if len(renames) in failing_renames: + raise PermissionError(13, "simulated rename failure") + real_replace(src, dst) + + monkeypatch.setattr(manager_module.os, "replace", replace) + + with pytest.raises(PermissionError, match="simulated rename failure"): + manager.update("magic-test") + + monkeypatch.setattr(manager_module.os, "replace", real_replace) + info = InstallationMetadata.load_from_dir(manager.installation_dir).extensions[ + "magic-test" + ] + assert (info.source, info.enabled) == (str(source_dir), False) + if failing_renames == {2, 3}: + assert not install_path.exists() + [staging_dir] = _staging_dirs(manager) + assert (staging_dir / "previous" / "extension.json").exists() + assert not (staging_dir / "previous" / "new-version").exists() + else: + assert (install_path / "extension.json").exists() + assert not (install_path / "new-version").exists() + assert _staging_dirs(manager) == [] + + +def test_previous_version_is_deleted_after_the_lock_is_released( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Deleting the previous version takes time proportional to its size, so + it must happen outside the lock: another session can open meanwhile. + """ + _install_disabled_with_new_version(manager, source_dir) + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + real_rmtree = shutil.rmtree + probes: list[list[BaseException]] = [] + + def rmtree(path, *args, **kwargs): + if Path(path).name.startswith(".magic-test.installing-"): + probes.append( + _run_threads( + ( + "probe", + lambda: InstallationMetadata.open( + manager.installation_dir + ).__exit__(None, None, None), + ) + ) + ) + return real_rmtree(path, *args, **kwargs) + + monkeypatch.setattr(manager_module.shutil, "rmtree", rmtree) + + manager.update("magic-test") + + assert probes == [[]] + + +def test_install_does_not_replace_a_directory_that_appeared_meanwhile( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Without force, an install that finds the extension installed by the + time it takes the lock (another install finished first) must fail, not + swap the other install's directory away. + """ + install_path = manager.installation_dir / "magic-test" + real_copytree = shutil.copytree + + def copytree(src, dst, *args, **kwargs): + if not install_path.exists(): + install_path.mkdir(parents=True) + (install_path / "winner").write_text("1") + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(manager_module.shutil, "copytree", copytree) + + with pytest.raises(FileExistsError): + manager.install(str(source_dir)) + + assert (install_path / "winner").exists() + assert not (install_path / "extension.json").exists() + assert _staging_dirs(manager) == [] diff --git a/tests/sdk/extensions/installation/test_installation_manager.py b/tests/sdk/extensions/installation/test_installation_manager.py index 58d28f9cf0b..01fa4a56faf 100644 --- a/tests/sdk/extensions/installation/test_installation_manager.py +++ b/tests/sdk/extensions/installation/test_installation_manager.py @@ -1,5 +1,7 @@ +import os import shutil import subprocess +import sys from pathlib import Path from unittest.mock import patch @@ -11,6 +13,7 @@ InstallationInterface, InstallationManager, InstallationMetadata, + InstallationMetadataLoadError, ) @@ -594,3 +597,62 @@ def test_update_clears_requested_ref_to_track_latest( assert updated is not None assert updated.requested_ref is None assert updated.resolved_ref == "def456" + + +# ============================================================================ +# Unloadable Metadata Tests +# ============================================================================ + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() == 0, + reason="root bypasses file permissions", +) +def test_list_installed_keeps_unreadable_metadata( + manager: InstallationManager[MockExtension], + mock_extension_dir: Path, + mock_extension: MockExtension, +): + """list_installed() must not rewrite an unreadable file as local/enabled.""" + manager.install(str(mock_extension_dir)) + manager.disable(mock_extension.name) + metadata_path = InstallationMetadata.get_metadata_path(manager.installation_dir) + metadata_path.chmod(0) + try: + with pytest.raises(InstallationMetadataLoadError): + manager.list_installed() + finally: + metadata_path.chmod(0o644) + + info = manager.get(mock_extension.name) + assert info is not None + assert info.source == str(mock_extension_dir) + assert info.enabled is False + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() == 0, + reason="root bypasses file permissions", +) +def test_get_reads_a_read_only_installation_dir( + manager: InstallationManager[MockExtension], + mock_extension_dir: Path, + mock_extension: MockExtension, +): + """get() is read-only, so it must not need to create the lock file.""" + manager.install(str(mock_extension_dir)) + manager.disable(mock_extension.name) + lock_path = manager.installation_dir / ".metadata.lock" + lock_path.unlink(missing_ok=True) + manager.installation_dir.chmod(0o555) + try: + info = manager.get(mock_extension.name) + finally: + manager.installation_dir.chmod(0o755) + + assert info is not None + assert info.source == str(mock_extension_dir) + assert info.enabled is False + assert not lock_path.exists() diff --git a/tests/sdk/extensions/installation/test_installation_metadata.py b/tests/sdk/extensions/installation/test_installation_metadata.py index 18d6f62d33c..da0687f473d 100644 --- a/tests/sdk/extensions/installation/test_installation_metadata.py +++ b/tests/sdk/extensions/installation/test_installation_metadata.py @@ -1,4 +1,8 @@ import logging +import os +import shutil +import stat +import sys from pathlib import Path import pytest @@ -8,6 +12,7 @@ InstallationInfo, InstallationInterface, InstallationMetadata, + InstallationMetadataLoadError, ) @@ -316,6 +321,73 @@ def test_open_does_not_save_on_exception(tmp_path: Path): assert loaded.extensions == {} +@pytest.mark.parametrize( + "content", + [ + pytest.param(b"invalid json {", id="invalid-json"), + pytest.param(b'{"extensions": []}', id="wrong-field-type"), + pytest.param(b"[]", id="wrong-top-level-type"), + pytest.param(b"\xff\xfe not utf-8", id="not-utf-8"), + ], +) +def test_open_refuses_metadata_it_cannot_load(tmp_path: Path, content: bytes): + """A write session must not save its empty fallback over a broken file.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + metadata_path = InstallationMetadata.get_metadata_path(installation_dir) + metadata_path.write_bytes(content) + + with pytest.raises(InstallationMetadataLoadError) as exc_info: + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + session.sync() + + assert exc_info.value.metadata_path == metadata_path + assert str(metadata_path) in str(exc_info.value) + assert metadata_path.read_bytes() == content + + +def test_open_without_metadata_file_starts_empty(tmp_path: Path): + """A missing file is a fresh install, not an error.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + assert session.extensions == {} + session.sync() + + loaded = InstallationMetadata.load_from_dir(installation_dir) + assert "mock-extension" in loaded.extensions + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() == 0, + reason="root bypasses file permissions", +) +def test_open_refuses_unreadable_metadata(tmp_path: Path): + """An unreadable file raises instead of being replaced by an atomic save.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + metadata_path = InstallationMetadata.get_metadata_path(installation_dir) + content = b'{"extensions": {}}' + metadata_path.write_bytes(content) + metadata_path.chmod(0) + try: + with pytest.raises(InstallationMetadataLoadError): + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + session.sync() + finally: + metadata_path.chmod(0o644) + + assert metadata_path.read_bytes() == content + + # ============================================================================ # validate_tracked Tests # ============================================================================ @@ -368,3 +440,94 @@ def test_discover_untracked_skips_mismatched_manifest_name(tmp_path: Path): assert discovered == [] assert "some-ext" not in metadata.extensions + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.parametrize("existing_mode", [0o600, 0o644], ids=oct) +def test_save_to_dir_keeps_existing_file_mode(tmp_path: Path, existing_mode: int): + """A save keeps the file's mode whatever the umask: an owner-only file stays + owner-only (sources can carry URL credentials), and a file other local users + read is not narrowed under them.""" + metadata_path = InstallationMetadata.get_metadata_path(tmp_path) + metadata_path.write_text("{}") + metadata_path.chmod(existing_mode) + previous_umask = os.umask(0o027) + try: + InstallationMetadata().save_to_dir(tmp_path) + finally: + os.umask(previous_umask) + + assert stat.S_IMODE(metadata_path.stat().st_mode) == existing_mode + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.parametrize( + ("umask", "expected"), + [ + pytest.param(0o022, 0o644, id="umask-022"), + pytest.param(0o077, 0o600, id="umask-077"), + ], +) +def test_save_to_dir_creates_new_file_with_umask_mode( + tmp_path: Path, umask: int, expected: int +): + """A new file gets the umask's mode, as a plain write would give it: a + strict umask keeps sources, which can carry URL credentials, owner-only.""" + metadata_path = InstallationMetadata.get_metadata_path(tmp_path) + previous_umask = os.umask(umask) + try: + InstallationMetadata().save_to_dir(tmp_path) + finally: + os.umask(previous_umask) + + assert stat.S_IMODE(metadata_path.stat().st_mode) == expected + + +# A uid and gid that are not root's: the user a root process (an agent-server +# container) shares an installation directory with. +_USER = 65534 + +_requires_root = pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() != 0, reason="changing owners needs root" +) + + +@_requires_root +def test_root_session_keeps_the_directory_users_ownership(tmp_path: Path): + """When root saves in a user's installation directory, that user can still + read the metadata file and take the lock afterwards.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + shutil.chown(installation_dir, _USER, _USER) + metadata_path = InstallationMetadata.get_metadata_path(installation_dir) + metadata_path.write_text('{"extensions": {}}') + metadata_path.chmod(0o600) + shutil.chown(metadata_path, _USER, _USER) + + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + session.sync() + + for path in (metadata_path, installation_dir / ".metadata.lock"): + assert (path.stat().st_uid, path.stat().st_gid) == (_USER, _USER), path + assert stat.S_IMODE(metadata_path.stat().st_mode) == 0o600 + + +@_requires_root +def test_root_session_does_not_give_away_a_hard_linked_lock_file(tmp_path: Path): + """A lock file that is another file's second name is left alone: the + directory's owner must not be able to get that file through root.""" + installation_dir = tmp_path / "installed" + installation_dir.mkdir() + shutil.chown(installation_dir, _USER, _USER) + other_file = tmp_path / "root-owned" + other_file.write_text("x") + os.link(other_file, installation_dir / ".metadata.lock") + + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ): + pass + + assert other_file.stat().st_uid == 0 diff --git a/tests/sdk/utils/test_files.py b/tests/sdk/utils/test_files.py index ec5796dcea9..51d584fbaeb 100644 --- a/tests/sdk/utils/test_files.py +++ b/tests/sdk/utils/test_files.py @@ -1,14 +1,22 @@ +import logging import os +import stat +import sys from pathlib import Path import pytest from openhands.sdk.utils import files -from openhands.sdk.utils.files import atomic_write_text, is_temp_file +from openhands.sdk.utils.files import ( + atomic_write_text, + is_temp_file, + read_text_retrying, +) +@pytest.mark.parametrize("mode", [0o600, None], ids=["mkstemp", "umask"]) def test_atomic_write_text_temp_file_is_marked_as_a_temp_file( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: int | None ) -> None: temp_paths: list[Path] = [] real_replace = os.replace @@ -21,7 +29,7 @@ def recording_replace(src, dst): names = ("base_state.json", "event-00000-abc.json", ".eventlog-len-3.marker") for name in names: - atomic_write_text(tmp_path / name, "{}") + atomic_write_text(tmp_path / name, "{}", mode=mode) assert [path.parent for path in temp_paths] == [tmp_path] * len(names) for name, temp_path in zip(names, temp_paths, strict=True): @@ -31,6 +39,49 @@ def recording_replace(src, dst): assert sorted(path.name for path in tmp_path.iterdir()) == sorted(names) +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.parametrize( + ("umask", "mode", "expected"), + [ + pytest.param(0o022, 0o600, 0o600, id="default-owner-only"), + pytest.param(0o022, 0o640, 0o640, id="explicit"), + pytest.param(0o022, None, 0o644, id="umask-022"), + pytest.param(0o077, None, 0o600, id="umask-077"), + ], +) +def test_atomic_write_text_mode( + tmp_path: Path, umask: int, mode: int | None, expected: int +) -> None: + path = tmp_path / "data.json" + path.write_text("old") + path.chmod(0o604) + previous_umask = os.umask(umask) + try: + atomic_write_text(path, "new", mode=mode) + finally: + os.umask(previous_umask) + + assert stat.S_IMODE(path.stat().st_mode) == expected + assert path.read_text() == "new" + + +@pytest.mark.skipif( + sys.platform == "win32" or getattr(os, "geteuid", lambda: -1)() == 0, + reason="needs a user who may not give files to root", +) +def test_atomic_write_text_owner_is_best_effort( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + path = tmp_path / "data.json" + + with caplog.at_level(logging.WARNING): + atomic_write_text(path, "new", owner=(0, 0)) + + assert path.read_text() == "new" + assert path.stat().st_uid == tmp_path.stat().st_uid != 0 + assert any("Could not keep the owner" in r.message for r in caplog.records) + + @pytest.mark.parametrize( "name", [ @@ -61,3 +112,150 @@ def test_is_temp_file_matches_tmp_names(name: str) -> None: ) def test_is_temp_file_ignores_other_names(name: str) -> None: assert not is_temp_file(Path(name)) + + +def _replace_failing(times: int, error: OSError, calls: list[Path]): + """An ``os.replace`` that raises ``error`` on its first ``times`` calls.""" + real_replace = os.replace + + def replace(src, dst): + calls.append(Path(dst)) + if len(calls) <= times: + raise error + real_replace(src, dst) + + return replace + + +@pytest.mark.parametrize("failures", [1, len(files._PERMISSION_ERROR_RETRY_DELAYS)]) +def test_atomic_write_text_retries_replace_blocked_by_an_open_handle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failures: int +) -> None: + """On Windows, a replace that meets a reader's open handle is retried.""" + path = tmp_path / "data.json" + path.write_text("old") + calls: list[Path] = [] + sleeps: list[float] = [] + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", True) + monkeypatch.setattr(files.time, "sleep", sleeps.append) + monkeypatch.setattr( + files.os, "replace", _replace_failing(failures, PermissionError(13, "x"), calls) + ) + + atomic_write_text(path, "new") + + assert path.read_text() == "new" + assert len(calls) == failures + 1 + assert sleeps == list(files._PERMISSION_ERROR_RETRY_DELAYS[:failures]) + assert [p.name for p in tmp_path.iterdir()] == ["data.json"] + + +def test_atomic_write_text_raises_when_replace_stays_blocked( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The retry is bounded: past the last delay the error is raised, the + destination is unchanged, and the temp file is removed. + """ + path = tmp_path / "data.json" + path.write_text("old") + calls: list[Path] = [] + sleeps: list[float] = [] + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", True) + monkeypatch.setattr(files.time, "sleep", sleeps.append) + monkeypatch.setattr( + files.os, + "replace", + _replace_failing( + len(files._PERMISSION_ERROR_RETRY_DELAYS) + 1, + PermissionError(13, "x"), + calls, + ), + ) + + with pytest.raises(PermissionError): + atomic_write_text(path, "new") + + assert len(calls) == len(files._PERMISSION_ERROR_RETRY_DELAYS) + 1 + assert sleeps == list(files._PERMISSION_ERROR_RETRY_DELAYS) + assert path.read_text() == "old" + assert [p.name for p in tmp_path.iterdir()] == ["data.json"] + + +@pytest.mark.parametrize( + ("retry_enabled", "error"), + [ + (False, PermissionError(13, "x")), + (True, OSError(28, "No space left on device")), + ], + ids=["permission_error_off_windows", "other_os_error_on_windows"], +) +def test_atomic_write_text_does_not_retry_other_replace_failures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + retry_enabled: bool, + error: OSError, +) -> None: + """Only a ``PermissionError`` on Windows is retried; anything else, which + a retry cannot fix, is raised at once. + """ + path = tmp_path / "data.json" + calls: list[Path] = [] + sleeps: list[float] = [] + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", retry_enabled) + monkeypatch.setattr(files.time, "sleep", sleeps.append) + monkeypatch.setattr(files.os, "replace", _replace_failing(1, error, calls)) + + with pytest.raises(type(error)): + atomic_write_text(path, "new") + + assert len(calls) == 1 + assert sleeps == [] + assert list(tmp_path.iterdir()) == [] + + +_MAX_RETRIES = len(files._PERMISSION_ERROR_RETRY_DELAYS) + + +@pytest.mark.parametrize( + ("retry_enabled", "failures", "reads", "succeeds"), + [ + (True, 1, 2, True), + (True, _MAX_RETRIES, _MAX_RETRIES + 1, True), + (True, _MAX_RETRIES + 1, _MAX_RETRIES + 1, False), + (False, 1, 1, False), + ], + ids=["one_failure", "max_failures", "too_many_failures", "off_windows"], +) +def test_read_text_retrying_retries_a_read_blocked_by_a_replace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + retry_enabled: bool, + failures: int, + reads: int, + succeeds: bool, +) -> None: + """On Windows, a read that meets a concurrent replace fails with + ``PermissionError`` and is retried, up to the last delay. + """ + path = tmp_path / "data.json" + path.write_text("content", encoding="utf-8") + calls: list[Path] = [] + real_read_text = Path.read_text + + def read_text(self: Path, *args, **kwargs) -> str: + calls.append(self) + if len(calls) <= failures: + raise PermissionError(13, "Permission denied") + return real_read_text(self, *args, **kwargs) + + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", retry_enabled) + monkeypatch.setattr(files.time, "sleep", lambda delay: None) + monkeypatch.setattr(Path, "read_text", read_text) + + if succeeds: + assert read_text_retrying(path) == "content" + else: + with pytest.raises(PermissionError): + read_text_retrying(path) + + assert calls == [path] * reads