From 08f46255d07d846bef8773d2c565aaa6ee9b3ad8 Mon Sep 17 00:00:00 2001 From: adithya Date: Wed, 7 Oct 2026 22:02:28 +0530 Subject: [PATCH 01/15] fix(extensions): lock and atomically write installation metadata --- .../sdk/extensions/installation/metadata.py | 46 +++- .../test_installation_concurrency.py | 223 ++++++++++++++++++ 2 files changed, 262 insertions(+), 7 deletions(-) create mode 100644 tests/sdk/extensions/installation/test_installation_concurrency.py diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 3dc737bcc34..1907319c508 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -2,8 +2,9 @@ from pathlib import Path from types import TracebackType -from typing import Any, ClassVar +from typing import Any, ClassVar, Final +from filelock import BaseFileLock, FileLock, Timeout from pydantic import BaseModel, Field, model_validator from openhands.sdk.extensions.installation.info import InstallationInfo @@ -12,10 +13,13 @@ ) from openhands.sdk.extensions.installation.utils import validate_extension_name from openhands.sdk.logger import get_logger +from openhands.sdk.utils.files import atomic_write_text logger = get_logger(__name__) +_LOCK_TIMEOUT_SECONDS: Final[float] = 30.0 + class MetadataSession: """Context manager that binds ``InstallationMetadata`` to its directory. @@ -32,10 +36,12 @@ def __init__( installed_dir: Path, metadata: InstallationMetadata, interface: InstallationInterface | None = None, + lock: BaseFileLock | None = None, ) -> None: self.installed_dir = installed_dir self.metadata = metadata self.interface = interface + self.lock = lock @property def extensions(self) -> dict[str, InstallationInfo]: @@ -73,8 +79,12 @@ def __exit__( exc_val: BaseException | None, exc_tb: TracebackType | None, ) -> None: - if exc_type is None: - self.metadata.save_to_dir(self.installed_dir) + try: + if exc_type is None: + self.metadata.save_to_dir(self.installed_dir) + finally: + if self.lock is not None: + self.lock.release() class InstallationMetadata(BaseModel): @@ -128,14 +138,36 @@ def open( ) -> MetadataSession: """Load metadata and return a session that auto-saves on exit. + Acquires a file lock on ``installed_dir`` before loading and holds it + until the session exits, so a load-modify-save cycle cannot interleave + with another one (another thread or process). Always use the result in + a ``with`` block, and do not nest sessions on the same directory. + Args: installed_dir: Root directory where extensions are installed. interface: Optional installation interface, required if the session will call ``sync()``. """ - return MetadataSession( - installed_dir, cls.load_from_dir(installed_dir), interface - ) + installed_dir.mkdir(parents=True, exist_ok=True) + lock = FileLock(installed_dir / ".metadata.lock") + try: + lock.acquire(timeout=_LOCK_TIMEOUT_SECONDS) + except Timeout: + logger.error( + f"Failed to acquire installation metadata lock in " + f"{_LOCK_TIMEOUT_SECONDS}s: {installed_dir}" + ) + raise TimeoutError( + "Installation metadata lock acquisition timed out after " + f"{_LOCK_TIMEOUT_SECONDS}s" + ) + try: + return MetadataSession( + installed_dir, cls.load_from_dir(installed_dir), interface, lock=lock + ) + except BaseException: + lock.release() + raise @classmethod def get_metadata_path(cls, installed_dir: Path) -> Path: @@ -159,7 +191,7 @@ def save_to_dir(self, installed_dir: Path) -> None: """Save metadata to the installed extensions directory.""" metadata_path = self.get_metadata_path(installed_dir) metadata_path.parent.mkdir(parents=True, exist_ok=True) - metadata_path.write_text(self.model_dump_json(indent=2), encoding="utf-8") + atomic_write_text(metadata_path, self.model_dump_json(indent=2), mode=0o644) def validate_tracked(self, installed_dir: Path) -> list[InstallationInfo]: """Validate tracked extensions exist on disk. diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py new file mode 100644 index 00000000000..78b38504fb6 --- /dev/null +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -0,0 +1,223 @@ +"""Concurrency regression tests for installation metadata""" + +import json +import os +import threading +from collections.abc import Callable +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from openhands.sdk.extensions.installation import ( + InstallationInterface, + InstallationManager, + InstallationMetadata, + MetadataSession, + metadata as metadata_module, +) +from openhands.sdk.utils import files as files_module + + +class MockExtension(BaseModel): + name: str + version: str = "0.0.1" + description: str = "Mock extension" + + +class MockExtensionInstallationInterface(InstallationInterface): + @staticmethod + def load_from_dir(extension_dir: Path) -> MockExtension: + return MockExtension.model_validate_json( + (extension_dir / "extension.json").read_text() + ) + + +@pytest.fixture +def source_dir(tmp_path: Path) -> Path: + directory = tmp_path / "src" / "magic-test" + directory.mkdir(parents=True) + (directory / "extension.json").write_text( + MockExtension(name="magic-test").model_dump_json() + ) + return directory + + +@pytest.fixture +def installation_dir(tmp_path: Path) -> Path: + directory = tmp_path / "installed" + directory.mkdir() + return directory + + +@pytest.fixture +def manager(installation_dir: Path) -> InstallationManager[MockExtension]: + return InstallationManager( + installation_dir=installation_dir, + installation_interface=MockExtensionInstallationInterface(), + ) + + +def _run_threads(*targets: tuple[str, Callable[[], object]]) -> list[BaseException]: + """Run each target in a named daemon thread; return exceptions they raised.""" + errors: list[BaseException] = [] + + def capture(fn: Callable[[], object]) -> Callable[[], None]: + def run() -> None: + try: + fn() + except BaseException as e: + errors.append(e) + + return run + + threads = [ + threading.Thread(target=capture(fn), name=name, daemon=True) + for name, fn in targets + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(10) + assert not thread.is_alive(), f"thread {thread.name!r} did not finish" + return errors + + +def _source_of(installation_dir: Path, name: str) -> str: + data = json.loads( + InstallationMetadata.get_metadata_path(installation_dir).read_text() + ) + return data["extensions"][name]["source"] + + +def test_list_cannot_load_metadata_while_install_session_is_open( + manager: InstallationManager[MockExtension], + installation_dir: Path, + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Lost update: a list call that loads metadata before an install saves it + later overwrites the install's entry with a ``source="local"`` rediscovery. + The session lock must keep the list call from loading until install saves. + """ + original_load = InstallationMetadata.load_from_dir.__func__ + install_loading = threading.Event() + list_loaded = threading.Event() + + def load(cls: type[InstallationMetadata], directory: Path): + me = threading.current_thread().name + if me == "install": + install_loading.set() + assert not list_loaded.wait(0.5), ( + "list_installed loaded metadata while install held its session" + ) + result = original_load(cls, directory) + if me == "list": + list_loaded.set() + return result + + monkeypatch.setattr(InstallationMetadata, "load_from_dir", classmethod(load)) + + def list_once_install_is_loading() -> None: + assert install_loading.wait(5) + manager.list_installed() + + errors = _run_threads( + ("install", lambda: manager.install(str(source_dir))), + ("list", list_once_install_is_loading), + ) + + assert errors == [] + assert _source_of(installation_dir, "magic-test") == str(source_dir) + + +def test_unlocked_reader_never_sees_partial_metadata_during_save( + manager: InstallationManager[MockExtension], + installation_dir: Path, + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Torn read: ``get()`` reads without the session lock, so a save must swap + in a complete file rather than truncate and rewrite it in place. + """ + manager.install(str(source_dir)) + metadata_path = InstallationMetadata.get_metadata_path(installation_dir) + seen_mid_save = [] + real_replace = os.replace + + def replace(src, dst) -> None: + if Path(dst) == metadata_path: + seen_mid_save.append(manager.get("magic-test")) + real_replace(src, dst) + + monkeypatch.setattr(files_module.os, "replace", replace) + + manager.install(str(source_dir), force=True) + + assert seen_mid_save, "save_to_dir did not write through an atomic replace" + info = seen_mid_save[0] + assert info is not None + assert info.source == str(source_dir) + + +def test_session_lock_times_out_instead_of_blocking_forever( + installation_dir: Path, monkeypatch: pytest.MonkeyPatch +): + """A session that is never closed must not hang every other caller.""" + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + + with InstallationMetadata.open(installation_dir): + errors = _run_threads( + ("other", lambda: InstallationMetadata.open(installation_dir)), + ) + + assert len(errors) == 1 + assert isinstance(errors[0], TimeoutError) + + +def _raise_runtime_error(*args, **kwargs): + raise RuntimeError("boom") + + +@pytest.mark.parametrize( + "failure", + ["error_inside_session", "save_fails", "load_fails"], +) +def test_lock_is_released_when_a_session_fails( + installation_dir: Path, monkeypatch: pytest.MonkeyPatch, failure: str +): + """A failing session must release the lock deterministically. + + The session and ``excinfo`` are kept referenced, as a caller that logs or + re-raises the exception would keep them. filelock also releases on garbage + collection, so without those references a missing release goes unnoticed. + """ + sessions: list[MetadataSession] = [] + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + + with monkeypatch.context() as patch: + if failure == "save_fails": + patch.setattr(InstallationMetadata, "save_to_dir", _raise_runtime_error) + elif failure == "load_fails": + patch.setattr( + InstallationMetadata, + "load_from_dir", + classmethod(_raise_runtime_error), + ) + with pytest.raises(RuntimeError) as excinfo: + with InstallationMetadata.open(installation_dir) as session: + sessions.append(session) + if failure == "error_inside_session": + raise RuntimeError("boom") + + errors = _run_threads( + ( + "next", + lambda: InstallationMetadata.open(installation_dir).__exit__( + None, None, None + ), + ), + ) + assert errors == [] + assert excinfo.value is not None + assert len(sessions) == (0 if failure == "load_fails" else 1) From 275423fe7ef0fbf792fa1158555d5a34e9efefd8 Mon Sep 17 00:00:00 2001 From: adithya Date: Fri, 9 Oct 2026 19:10:03 +0530 Subject: [PATCH 02/15] fix(extensions): read installation metadata under the lock --- .../canvas_extensions/installed.py | 2 +- .../sdk/extensions/installation/manager.py | 2 +- .../sdk/extensions/installation/metadata.py | 35 +++++++-- .../test_installation_concurrency.py | 74 ++++++++++++++----- 4 files changed, 86 insertions(+), 27 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py index c41098d4438..3fba71125e2 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py @@ -87,7 +87,7 @@ def _tracked_names(installed_dir: Path) -> set[str]: """ if not installed_dir.exists(): return set() - metadata = InstallationMetadata.load_from_dir(installed_dir) + metadata = InstallationMetadata.read(installed_dir) return {info.name for info in metadata.validate_tracked(installed_dir)} diff --git a/openhands-sdk/openhands/sdk/extensions/installation/manager.py b/openhands-sdk/openhands/sdk/extensions/installation/manager.py index 348511e3e36..6f65f4eab81 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/manager.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/manager.py @@ -281,7 +281,7 @@ def get(self, name: str) -> InstallationInfo | None: """ validate_extension_name(name) - metadata = InstallationMetadata.load_from_dir(self.installation_dir) + metadata = InstallationMetadata.read(self.installation_dir) info = metadata.extensions.get(name) if info is not None: diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 1907319c508..3baf35ea093 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -149,6 +149,33 @@ def open( session will call ``sync()``. """ installed_dir.mkdir(parents=True, exist_ok=True) + lock = cls._acquire_lock(installed_dir) + try: + return MetadataSession( + installed_dir, cls.load_from_dir(installed_dir), interface, lock=lock + ) + except BaseException: + lock.release() + raise + + @classmethod + def read(cls, installed_dir: Path) -> InstallationMetadata: + """Load metadata under the session lock, without saving. + + For read-only callers. Holding the lock keeps the read from overlapping + a save, whose atomic replace fails on Windows while another handle has + the file open. + """ + if not cls.get_metadata_path(installed_dir).exists(): + return cls() + lock = cls._acquire_lock(installed_dir) + try: + return cls.load_from_dir(installed_dir) + finally: + lock.release() + + @staticmethod + def _acquire_lock(installed_dir: Path) -> BaseFileLock: lock = FileLock(installed_dir / ".metadata.lock") try: lock.acquire(timeout=_LOCK_TIMEOUT_SECONDS) @@ -161,13 +188,7 @@ def open( "Installation metadata lock acquisition timed out after " f"{_LOCK_TIMEOUT_SECONDS}s" ) - try: - return MetadataSession( - installed_dir, cls.load_from_dir(installed_dir), interface, lock=lock - ) - except BaseException: - lock.release() - raise + return lock @classmethod def get_metadata_path(cls, installed_dir: Path) -> Path: diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py index 78b38504fb6..cae95835901 100644 --- a/tests/sdk/extensions/installation/test_installation_concurrency.py +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -90,26 +90,32 @@ def _source_of(installation_dir: Path, name: str) -> str: return data["extensions"][name]["source"] -def test_list_cannot_load_metadata_while_install_session_is_open( +@pytest.mark.parametrize("operation", ["install", "disable"]) +def test_list_cannot_load_metadata_while_another_session_is_open( manager: InstallationManager[MockExtension], installation_dir: Path, source_dir: Path, monkeypatch: pytest.MonkeyPatch, + operation: str, ): - """Lost update: a list call that loads metadata before an install saves it - later overwrites the install's entry with a ``source="local"`` rediscovery. - The session lock must keep the list call from loading until install saves. + """Lost update: a list call that loads metadata before another session + saves later overwrites that session's change with a stale copy (for a fresh + install, plus a ``source="local"`` rediscovery). The session lock must keep + the list call from loading until the other session has saved. """ + if operation == "disable": + manager.install(str(source_dir)) + original_load = InstallationMetadata.load_from_dir.__func__ - install_loading = threading.Event() + writer_loading = threading.Event() list_loaded = threading.Event() def load(cls: type[InstallationMetadata], directory: Path): me = threading.current_thread().name - if me == "install": - install_loading.set() + if me == "writer": + writer_loading.set() assert not list_loaded.wait(0.5), ( - "list_installed loaded metadata while install held its session" + "list_installed loaded metadata while another session was open" ) result = original_load(cls, directory) if me == "list": @@ -118,17 +124,25 @@ def load(cls: type[InstallationMetadata], directory: Path): monkeypatch.setattr(InstallationMetadata, "load_from_dir", classmethod(load)) - def list_once_install_is_loading() -> None: - assert install_loading.wait(5) + def write() -> None: + if operation == "install": + manager.install(str(source_dir)) + else: + manager.disable("magic-test") + + def list_once_writer_is_loading() -> None: + assert writer_loading.wait(5) manager.list_installed() - errors = _run_threads( - ("install", lambda: manager.install(str(source_dir))), - ("list", list_once_install_is_loading), - ) + errors = _run_threads(("writer", write), ("list", list_once_writer_is_loading)) assert errors == [] - assert _source_of(installation_dir, "magic-test") == str(source_dir) + data = json.loads( + InstallationMetadata.get_metadata_path(installation_dir).read_text() + ) + entry = data["extensions"]["magic-test"] + assert entry["source"] == str(source_dir) + assert entry["enabled"] is (operation == "install") def test_unlocked_reader_never_sees_partial_metadata_during_save( @@ -137,8 +151,9 @@ def test_unlocked_reader_never_sees_partial_metadata_during_save( source_dir: Path, monkeypatch: pytest.MonkeyPatch, ): - """Torn read: ``get()`` reads without the session lock, so a save must swap - in a complete file rather than truncate and rewrite it in place. + """Torn read: a reader that does not take the lock (such as code reading + ``.installed.json`` directly) must never see a truncated file, so a save + must swap in a complete file rather than rewrite it in place. """ manager.install(str(source_dir)) metadata_path = InstallationMetadata.get_metadata_path(installation_dir) @@ -147,7 +162,8 @@ def test_unlocked_reader_never_sees_partial_metadata_during_save( def replace(src, dst) -> None: if Path(dst) == metadata_path: - seen_mid_save.append(manager.get("magic-test")) + loaded = InstallationMetadata.load_from_dir(installation_dir) + seen_mid_save.append(loaded.extensions.get("magic-test")) real_replace(src, dst) monkeypatch.setattr(files_module.os, "replace", replace) @@ -160,6 +176,28 @@ def replace(src, dst) -> None: assert info.source == str(source_dir) +def test_get_waits_for_an_open_session( + manager: InstallationManager[MockExtension], + installation_dir: Path, + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """``get()`` reads under the lock, so it can never hold the file open + while a save replaces it (``os.replace`` fails on Windows in that case). + """ + manager.install(str(source_dir)) + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + + with InstallationMetadata.open(installation_dir): + errors = _run_threads(("get", lambda: manager.get("magic-test"))) + + assert len(errors) == 1 + assert isinstance(errors[0], TimeoutError) + info = manager.get("magic-test") + assert info is not None + assert info.source == str(source_dir) + + def test_session_lock_times_out_instead_of_blocking_forever( installation_dir: Path, monkeypatch: pytest.MonkeyPatch ): From b917f9f6209111843374ee1a2e9a7fee37799ef0 Mon Sep 17 00:00:00 2001 From: adithya Date: Wed, 14 Oct 2026 07:25:48 +0530 Subject: [PATCH 03/15] fix(sdk): write installation metadata with owner-only permissions --- .../sdk/extensions/installation/metadata.py | 2 +- .../installation/test_installation_metadata.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 3baf35ea093..e272e3a259d 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -212,7 +212,7 @@ def save_to_dir(self, installed_dir: Path) -> None: """Save metadata to the installed extensions directory.""" metadata_path = self.get_metadata_path(installed_dir) metadata_path.parent.mkdir(parents=True, exist_ok=True) - atomic_write_text(metadata_path, self.model_dump_json(indent=2), mode=0o644) + atomic_write_text(metadata_path, self.model_dump_json(indent=2), mode=0o600) def validate_tracked(self, installed_dir: Path) -> list[InstallationInfo]: """Validate tracked extensions exist on disk. diff --git a/tests/sdk/extensions/installation/test_installation_metadata.py b/tests/sdk/extensions/installation/test_installation_metadata.py index 18d6f62d33c..dc8f08dd2a0 100644 --- a/tests/sdk/extensions/installation/test_installation_metadata.py +++ b/tests/sdk/extensions/installation/test_installation_metadata.py @@ -1,4 +1,6 @@ import logging +import stat +import sys from pathlib import Path import pytest @@ -368,3 +370,16 @@ def test_discover_untracked_skips_mismatched_manifest_name(tmp_path: Path): assert discovered == [] assert "some-ext" not in metadata.extensions + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +def test_save_to_dir_writes_owner_only_file(tmp_path: Path): + """Sources can carry URL credentials, so the metadata file must not be + readable by group or other users, even if an existing file was wider.""" + metadata_path = InstallationMetadata.get_metadata_path(tmp_path) + metadata_path.write_text("{}") + metadata_path.chmod(0o644) + + InstallationMetadata().save_to_dir(tmp_path) + + assert stat.S_IMODE(metadata_path.stat().st_mode) == 0o600 From 6e922e36bee30c5a27d29598b9d4cfc552714c20 Mon Sep 17 00:00:00 2001 From: adithya Date: Tue, 10 Nov 2026 14:06:41 +0530 Subject: [PATCH 04/15] fix(sdk): refuse to save over installation metadata that cannot be loaded --- .../sdk/extensions/installation/__init__.py | 2 + .../sdk/extensions/installation/metadata.py | 46 ++++++++++++- .../test_installation_concurrency.py | 6 +- .../installation/test_installation_manager.py | 35 ++++++++++ .../test_installation_metadata.py | 69 +++++++++++++++++++ 5 files changed, 154 insertions(+), 4 deletions(-) diff --git a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py index 68403c5b198..5e8aba19905 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py @@ -6,6 +6,7 @@ from openhands.sdk.extensions.installation.manager import InstallationManager from openhands.sdk.extensions.installation.metadata import ( InstallationMetadata, + InstallationMetadataLoadError, MetadataSession, ) @@ -16,5 +17,6 @@ "ExtensionProtocol", "InstallationManager", "InstallationMetadata", + "InstallationMetadataLoadError", "MetadataSession", ] diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index e272e3a259d..2a318dc8c19 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -21,6 +21,22 @@ _LOCK_TIMEOUT_SECONDS: Final[float] = 30.0 +class InstallationMetadataLoadError(Exception): + """An existing metadata file could not be read or parsed. + + A write session raises this instead of starting from empty metadata, + because saving that fallback would overwrite every recorded source and + enabled state. The file is left unchanged. + """ + + def __init__(self, metadata_path: Path, cause: BaseException) -> None: + super().__init__( + f"Could not load installation metadata at {metadata_path} " + f"({type(cause).__name__}); the file was left unchanged" + ) + self.metadata_path = metadata_path + + class MetadataSession: """Context manager that binds ``InstallationMetadata`` to its directory. @@ -147,12 +163,20 @@ def open( installed_dir: Root directory where extensions are installed. interface: Optional installation interface, required if the session will call ``sync()``. + + Raises: + InstallationMetadataLoadError: The metadata file exists but cannot + be read or parsed. The lock is released and the file is left + unchanged. """ installed_dir.mkdir(parents=True, exist_ok=True) lock = cls._acquire_lock(installed_dir) try: return MetadataSession( - installed_dir, cls.load_from_dir(installed_dir), interface, lock=lock + installed_dir, + cls._load_for_session(installed_dir), + interface, + lock=lock, ) except BaseException: lock.release() @@ -208,6 +232,26 @@ def load_from_dir(cls, installed_dir: Path) -> InstallationMetadata: logger.warning(f"Failed to load installed extension metadata: {e}") return cls() + @classmethod + def _load_for_session(cls, installed_dir: Path) -> InstallationMetadata: + """Load metadata for a write session. + + Unlike ``load_from_dir``, a file that exists but cannot be read or + parsed raises instead of loading as empty, since the session would + save that empty fallback over it. A missing file still loads as empty. + """ + metadata_path = cls.get_metadata_path(installed_dir) + try: + text = metadata_path.read_text(encoding="utf-8") + except FileNotFoundError: + return cls() + except (OSError, UnicodeDecodeError) as e: + raise InstallationMetadataLoadError(metadata_path, e) from e + try: + return cls.model_validate_json(text) + except Exception as e: + raise InstallationMetadataLoadError(metadata_path, e) from e + def save_to_dir(self, installed_dir: Path) -> None: """Save metadata to the installed extensions directory.""" metadata_path = self.get_metadata_path(installed_dir) diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py index cae95835901..d41151e0918 100644 --- a/tests/sdk/extensions/installation/test_installation_concurrency.py +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -106,7 +106,7 @@ def test_list_cannot_load_metadata_while_another_session_is_open( if operation == "disable": manager.install(str(source_dir)) - original_load = InstallationMetadata.load_from_dir.__func__ + original_load = InstallationMetadata._load_for_session.__func__ writer_loading = threading.Event() list_loaded = threading.Event() @@ -122,7 +122,7 @@ def load(cls: type[InstallationMetadata], directory: Path): list_loaded.set() return result - monkeypatch.setattr(InstallationMetadata, "load_from_dir", classmethod(load)) + monkeypatch.setattr(InstallationMetadata, "_load_for_session", classmethod(load)) def write() -> None: if operation == "install": @@ -239,7 +239,7 @@ def test_lock_is_released_when_a_session_fails( elif failure == "load_fails": patch.setattr( InstallationMetadata, - "load_from_dir", + "_load_for_session", classmethod(_raise_runtime_error), ) with pytest.raises(RuntimeError) as excinfo: diff --git a/tests/sdk/extensions/installation/test_installation_manager.py b/tests/sdk/extensions/installation/test_installation_manager.py index 58d28f9cf0b..22a99a2262c 100644 --- a/tests/sdk/extensions/installation/test_installation_manager.py +++ b/tests/sdk/extensions/installation/test_installation_manager.py @@ -1,5 +1,7 @@ +import os import shutil import subprocess +import sys from pathlib import Path from unittest.mock import patch @@ -11,6 +13,7 @@ InstallationInterface, InstallationManager, InstallationMetadata, + InstallationMetadataLoadError, ) @@ -594,3 +597,35 @@ def test_update_clears_requested_ref_to_track_latest( assert updated is not None assert updated.requested_ref is None assert updated.resolved_ref == "def456" + + +# ============================================================================ +# Unloadable Metadata Tests +# ============================================================================ + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() == 0, + reason="root bypasses file permissions", +) +def test_list_installed_keeps_unreadable_metadata( + manager: InstallationManager[MockExtension], + mock_extension_dir: Path, + mock_extension: MockExtension, +): + """list_installed() must not rewrite an unreadable file as local/enabled.""" + manager.install(str(mock_extension_dir)) + manager.disable(mock_extension.name) + metadata_path = InstallationMetadata.get_metadata_path(manager.installation_dir) + metadata_path.chmod(0) + try: + with pytest.raises(InstallationMetadataLoadError): + manager.list_installed() + finally: + metadata_path.chmod(0o644) + + info = manager.get(mock_extension.name) + assert info is not None + assert info.source == str(mock_extension_dir) + assert info.enabled is False diff --git a/tests/sdk/extensions/installation/test_installation_metadata.py b/tests/sdk/extensions/installation/test_installation_metadata.py index dc8f08dd2a0..3bf2e5c9728 100644 --- a/tests/sdk/extensions/installation/test_installation_metadata.py +++ b/tests/sdk/extensions/installation/test_installation_metadata.py @@ -1,4 +1,5 @@ import logging +import os import stat import sys from pathlib import Path @@ -10,6 +11,7 @@ InstallationInfo, InstallationInterface, InstallationMetadata, + InstallationMetadataLoadError, ) @@ -318,6 +320,73 @@ def test_open_does_not_save_on_exception(tmp_path: Path): assert loaded.extensions == {} +@pytest.mark.parametrize( + "content", + [ + pytest.param(b"invalid json {", id="invalid-json"), + pytest.param(b'{"extensions": []}', id="wrong-field-type"), + pytest.param(b"[]", id="wrong-top-level-type"), + pytest.param(b"\xff\xfe not utf-8", id="not-utf-8"), + ], +) +def test_open_refuses_metadata_it_cannot_load(tmp_path: Path, content: bytes): + """A write session must not save its empty fallback over a broken file.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + metadata_path = InstallationMetadata.get_metadata_path(installation_dir) + metadata_path.write_bytes(content) + + with pytest.raises(InstallationMetadataLoadError) as exc_info: + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + session.sync() + + assert exc_info.value.metadata_path == metadata_path + assert str(metadata_path) in str(exc_info.value) + assert metadata_path.read_bytes() == content + + +def test_open_without_metadata_file_starts_empty(tmp_path: Path): + """A missing file is a fresh install, not an error.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + assert session.extensions == {} + session.sync() + + loaded = InstallationMetadata.load_from_dir(installation_dir) + assert "mock-extension" in loaded.extensions + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() == 0, + reason="root bypasses file permissions", +) +def test_open_refuses_unreadable_metadata(tmp_path: Path): + """An unreadable file raises instead of being replaced by an atomic save.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + metadata_path = InstallationMetadata.get_metadata_path(installation_dir) + content = b'{"extensions": {}}' + metadata_path.write_bytes(content) + metadata_path.chmod(0) + try: + with pytest.raises(InstallationMetadataLoadError): + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + session.sync() + finally: + metadata_path.chmod(0o644) + + assert metadata_path.read_bytes() == content + + # ============================================================================ # validate_tracked Tests # ============================================================================ From 5c3c386cdd0686f30e22679790dacae0a9b35ca5 Mon Sep 17 00:00:00 2001 From: adithya Date: Tue, 10 Nov 2026 15:26:35 +0530 Subject: [PATCH 05/15] fix(sdk): read installation metadata without the lock; retry a blocked replace on Windows --- .../canvas_extensions/installed.py | 2 +- .../sdk/extensions/installation/manager.py | 2 +- .../sdk/extensions/installation/metadata.py | 23 ++--- openhands-sdk/openhands/sdk/utils/files.py | 23 ++++- .../test_installation_concurrency.py | 78 +++++++++------- .../installation/test_installation_manager.py | 27 ++++++ tests/sdk/utils/test_files.py | 93 +++++++++++++++++++ 7 files changed, 194 insertions(+), 54 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py index 3fba71125e2..c41098d4438 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py @@ -87,7 +87,7 @@ def _tracked_names(installed_dir: Path) -> set[str]: """ if not installed_dir.exists(): return set() - metadata = InstallationMetadata.read(installed_dir) + metadata = InstallationMetadata.load_from_dir(installed_dir) return {info.name for info in metadata.validate_tracked(installed_dir)} diff --git a/openhands-sdk/openhands/sdk/extensions/installation/manager.py b/openhands-sdk/openhands/sdk/extensions/installation/manager.py index 452dd5e5568..a6c54693587 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/manager.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/manager.py @@ -342,7 +342,7 @@ def get(self, name: str) -> InstallationInfo | None: """ validate_extension_name(name) - metadata = InstallationMetadata.read(self.installation_dir) + metadata = InstallationMetadata.load_from_dir(self.installation_dir) info = metadata.extensions.get(name) if info is not None: diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 2a318dc8c19..273852d5d6d 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -182,22 +182,6 @@ def open( lock.release() raise - @classmethod - def read(cls, installed_dir: Path) -> InstallationMetadata: - """Load metadata under the session lock, without saving. - - For read-only callers. Holding the lock keeps the read from overlapping - a save, whose atomic replace fails on Windows while another handle has - the file open. - """ - if not cls.get_metadata_path(installed_dir).exists(): - return cls() - lock = cls._acquire_lock(installed_dir) - try: - return cls.load_from_dir(installed_dir) - finally: - lock.release() - @staticmethod def _acquire_lock(installed_dir: Path) -> BaseFileLock: lock = FileLock(installed_dir / ".metadata.lock") @@ -221,7 +205,12 @@ def get_metadata_path(cls, installed_dir: Path) -> Path: @classmethod def load_from_dir(cls, installed_dir: Path) -> InstallationMetadata: - """Load metadata from the installed extensions directory.""" + """Load metadata from the installed extensions directory. + + For read-only callers. Takes no lock: a save swaps in a complete file, + so a concurrent read sees the old or the new version, never a partial + one. A file that cannot be read or parsed loads as empty. + """ metadata_path = cls.get_metadata_path(installed_dir) if not metadata_path.exists(): return cls() diff --git a/openhands-sdk/openhands/sdk/utils/files.py b/openhands-sdk/openhands/sdk/utils/files.py index cdf2b7e6807..d35f72cf91d 100644 --- a/openhands-sdk/openhands/sdk/utils/files.py +++ b/openhands-sdk/openhands/sdk/utils/files.py @@ -1,5 +1,6 @@ import os import tempfile +import time from pathlib import Path from typing import Final @@ -8,6 +9,14 @@ # to ``...tmp`` next to ``path`` and then renames it to ````. _TEMP_SUFFIX: Final[str] = ".tmp" +# On Windows, ``os.replace`` raises ``PermissionError`` while another handle +# has the destination open, because Python opens files without delete +# sharing. Readers hold the file only while reading it, so the replace is +# retried after these delays (about 0.9 s in total) before the error is raised. +# On other platforms a replace never fails that way, and it is not retried. +_RETRY_REPLACE_ON_PERMISSION_ERROR: Final[bool] = os.name == "nt" +_REPLACE_RETRY_DELAYS: Final[tuple[float, ...]] = (0.01, 0.02, 0.05, 0.1, 0.2, 0.5) + def is_temp_file(path: Path) -> bool: """Return True if ``path`` is named like a file that is still being written. @@ -18,6 +27,18 @@ def is_temp_file(path: Path) -> bool: return path.suffix == _TEMP_SUFFIX +def _replace(source: Path, destination: Path) -> None: + """``os.replace``, retried while a reader has ``destination`` open.""" + if _RETRY_REPLACE_ON_PERMISSION_ERROR: + for delay in _REPLACE_RETRY_DELAYS: + try: + os.replace(source, destination) + return + except PermissionError: + time.sleep(delay) + os.replace(source, destination) + + def atomic_write_text(path: Path, value: str, mode: int = 0o600) -> None: """Atomically write text with owner-only permissions.""" fd, temporary_name = tempfile.mkstemp( @@ -36,7 +57,7 @@ def atomic_write_text(path: Path, value: str, mode: int = 0o600) -> None: file.write(value) file.flush() os.fsync(file.fileno()) - os.replace(temporary_path, path) + _replace(temporary_path, path) except BaseException: if fd >= 0: try: diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py index d41151e0918..ccb217c557e 100644 --- a/tests/sdk/extensions/installation/test_installation_concurrency.py +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -1,7 +1,6 @@ """Concurrency regression tests for installation metadata""" import json -import os import threading from collections.abc import Callable from pathlib import Path @@ -10,13 +9,17 @@ from pydantic import BaseModel from openhands.sdk.extensions.installation import ( + InstallationInfo, InstallationInterface, InstallationManager, InstallationMetadata, MetadataSession, metadata as metadata_module, ) -from openhands.sdk.utils import files as files_module + + +# Disable/enable pairs the writer runs while the reader reads. +_SAVES = 100 class MockExtension(BaseModel): @@ -145,57 +148,64 @@ def list_once_writer_is_loading() -> None: assert entry["enabled"] is (operation == "install") -def test_unlocked_reader_never_sees_partial_metadata_during_save( +def test_reader_thread_never_sees_partial_metadata_during_saves( manager: InstallationManager[MockExtension], - installation_dir: Path, source_dir: Path, - monkeypatch: pytest.MonkeyPatch, ): - """Torn read: a reader that does not take the lock (such as code reading - ``.installed.json`` directly) must never see a truncated file, so a save - must swap in a complete file rather than rewrite it in place. + """Torn read: ``get()`` takes no lock, so while another thread saves it + must still see a complete file, never a truncated one (which loads as + empty). On Windows this also covers the writer: a save whose replace + meets a reader's open handle must retry instead of raising. """ manager.install(str(source_dir)) - metadata_path = InstallationMetadata.get_metadata_path(installation_dir) - seen_mid_save = [] - real_replace = os.replace - - def replace(src, dst) -> None: - if Path(dst) == metadata_path: - loaded = InstallationMetadata.load_from_dir(installation_dir) - seen_mid_save.append(loaded.extensions.get("magic-test")) - real_replace(src, dst) - - monkeypatch.setattr(files_module.os, "replace", replace) - - manager.install(str(source_dir), force=True) + saving = threading.Event() + done = threading.Event() + reads: list[InstallationInfo | None] = [] + + def save_repeatedly() -> None: + try: + for _ in range(_SAVES): + manager.disable("magic-test") + saving.set() + manager.enable("magic-test") + finally: + done.set() + + def read_until_done() -> None: + assert saving.wait(5) + while not done.is_set(): + reads.append(manager.get("magic-test")) + + errors = _run_threads(("writer", save_repeatedly), ("reader", read_until_done)) - assert seen_mid_save, "save_to_dir did not write through an atomic replace" - info = seen_mid_save[0] - assert info is not None - assert info.source == str(source_dir) + assert errors == [] + assert reads, "the reader never ran while the writer was saving" + torn = [info for info in reads if info is None or info.source != str(source_dir)] + assert torn == [], f"{len(torn)} of {len(reads)} reads saw partial metadata" -def test_get_waits_for_an_open_session( +def test_get_does_not_wait_for_an_open_session( manager: InstallationManager[MockExtension], installation_dir: Path, source_dir: Path, monkeypatch: pytest.MonkeyPatch, ): - """``get()`` reads under the lock, so it can never hold the file open - while a save replaces it (``os.replace`` fails on Windows in that case). + """``get()`` is read-only and takes no lock, so a session held open by + another caller must not block it or make it time out. """ manager.install(str(source_dir)) monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + results: list[InstallationInfo | None] = [] with InstallationMetadata.open(installation_dir): - errors = _run_threads(("get", lambda: manager.get("magic-test"))) + errors = _run_threads( + ("get", lambda: results.append(manager.get("magic-test"))) + ) - assert len(errors) == 1 - assert isinstance(errors[0], TimeoutError) - info = manager.get("magic-test") - assert info is not None - assert info.source == str(source_dir) + assert errors == [] + assert len(results) == 1 + assert results[0] is not None + assert results[0].source == str(source_dir) def test_session_lock_times_out_instead_of_blocking_forever( diff --git a/tests/sdk/extensions/installation/test_installation_manager.py b/tests/sdk/extensions/installation/test_installation_manager.py index 22a99a2262c..01fa4a56faf 100644 --- a/tests/sdk/extensions/installation/test_installation_manager.py +++ b/tests/sdk/extensions/installation/test_installation_manager.py @@ -629,3 +629,30 @@ def test_list_installed_keeps_unreadable_metadata( assert info is not None assert info.source == str(mock_extension_dir) assert info.enabled is False + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() == 0, + reason="root bypasses file permissions", +) +def test_get_reads_a_read_only_installation_dir( + manager: InstallationManager[MockExtension], + mock_extension_dir: Path, + mock_extension: MockExtension, +): + """get() is read-only, so it must not need to create the lock file.""" + manager.install(str(mock_extension_dir)) + manager.disable(mock_extension.name) + lock_path = manager.installation_dir / ".metadata.lock" + lock_path.unlink(missing_ok=True) + manager.installation_dir.chmod(0o555) + try: + info = manager.get(mock_extension.name) + finally: + manager.installation_dir.chmod(0o755) + + assert info is not None + assert info.source == str(mock_extension_dir) + assert info.enabled is False + assert not lock_path.exists() diff --git a/tests/sdk/utils/test_files.py b/tests/sdk/utils/test_files.py index ec5796dcea9..041be70927d 100644 --- a/tests/sdk/utils/test_files.py +++ b/tests/sdk/utils/test_files.py @@ -61,3 +61,96 @@ def test_is_temp_file_matches_tmp_names(name: str) -> None: ) def test_is_temp_file_ignores_other_names(name: str) -> None: assert not is_temp_file(Path(name)) + + +def _replace_failing(times: int, error: OSError, calls: list[Path]): + """An ``os.replace`` that raises ``error`` on its first ``times`` calls.""" + real_replace = os.replace + + def replace(src, dst): + calls.append(Path(dst)) + if len(calls) <= times: + raise error + real_replace(src, dst) + + return replace + + +@pytest.mark.parametrize("failures", [1, len(files._REPLACE_RETRY_DELAYS)]) +def test_atomic_write_text_retries_replace_blocked_by_an_open_handle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failures: int +) -> None: + """On Windows, a replace that meets a reader's open handle is retried.""" + path = tmp_path / "data.json" + path.write_text("old") + calls: list[Path] = [] + sleeps: list[float] = [] + monkeypatch.setattr(files, "_RETRY_REPLACE_ON_PERMISSION_ERROR", True) + monkeypatch.setattr(files.time, "sleep", sleeps.append) + monkeypatch.setattr( + files.os, "replace", _replace_failing(failures, PermissionError(13, "x"), calls) + ) + + atomic_write_text(path, "new") + + assert path.read_text() == "new" + assert len(calls) == failures + 1 + assert sleeps == list(files._REPLACE_RETRY_DELAYS[:failures]) + assert [p.name for p in tmp_path.iterdir()] == ["data.json"] + + +def test_atomic_write_text_raises_when_replace_stays_blocked( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The retry is bounded: past the last delay the error is raised, the + destination is unchanged, and the temp file is removed. + """ + path = tmp_path / "data.json" + path.write_text("old") + calls: list[Path] = [] + sleeps: list[float] = [] + monkeypatch.setattr(files, "_RETRY_REPLACE_ON_PERMISSION_ERROR", True) + monkeypatch.setattr(files.time, "sleep", sleeps.append) + monkeypatch.setattr( + files.os, "replace", _replace_failing(100, PermissionError(13, "x"), calls) + ) + + with pytest.raises(PermissionError): + atomic_write_text(path, "new") + + assert len(calls) == len(files._REPLACE_RETRY_DELAYS) + 1 + assert sleeps == list(files._REPLACE_RETRY_DELAYS) + assert path.read_text() == "old" + assert [p.name for p in tmp_path.iterdir()] == ["data.json"] + + +@pytest.mark.parametrize( + ("retry_enabled", "error"), + [ + (False, PermissionError(13, "x")), + (True, OSError(28, "No space left on device")), + ], + ids=["permission_error_off_windows", "other_os_error_on_windows"], +) +def test_atomic_write_text_does_not_retry_other_replace_failures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + retry_enabled: bool, + error: OSError, +) -> None: + """Only a ``PermissionError`` on Windows is retried; anything else, which + a retry cannot fix, is raised at once. + """ + path = tmp_path / "data.json" + calls: list[Path] = [] + sleeps: list[float] = [] + monkeypatch.setattr(files, "_RETRY_REPLACE_ON_PERMISSION_ERROR", retry_enabled) + monkeypatch.setattr(files.time, "sleep", sleeps.append) + monkeypatch.setattr(files.os, "replace", _replace_failing(1, error, calls)) + + with pytest.raises(type(error)): + atomic_write_text(path, "new") + + assert len(calls) == 1 + assert sleeps == [] + assert list(tmp_path.iterdir()) == [] From e88b6ff41f5362adf34823b88d64df87de66955a Mon Sep 17 00:00:00 2001 From: adithya Date: Tue, 10 Nov 2026 16:13:43 +0530 Subject: [PATCH 06/15] fix(sdk): retry an unlocked metadata read that meets a replace on Windows --- .../sdk/extensions/installation/metadata.py | 7 +- openhands-sdk/openhands/sdk/utils/files.py | 48 +++++++++---- .../test_installation_concurrency.py | 20 +++--- tests/sdk/utils/test_files.py | 68 ++++++++++++++++--- 4 files changed, 109 insertions(+), 34 deletions(-) diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 273852d5d6d..59c03f7c2da 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -13,7 +13,7 @@ ) from openhands.sdk.extensions.installation.utils import validate_extension_name from openhands.sdk.logger import get_logger -from openhands.sdk.utils.files import atomic_write_text +from openhands.sdk.utils.files import atomic_write_text, read_text_retrying logger = get_logger(__name__) @@ -209,14 +209,15 @@ def load_from_dir(cls, installed_dir: Path) -> InstallationMetadata: For read-only callers. Takes no lock: a save swaps in a complete file, so a concurrent read sees the old or the new version, never a partial - one. A file that cannot be read or parsed loads as empty. + one. On Windows, a read that meets the swap is retried. A file that + cannot be read or parsed loads as empty. """ metadata_path = cls.get_metadata_path(installed_dir) if not metadata_path.exists(): return cls() try: - return cls.model_validate_json(metadata_path.read_text(encoding="utf-8")) + return cls.model_validate_json(read_text_retrying(metadata_path)) except Exception as e: logger.warning(f"Failed to load installed extension metadata: {e}") return cls() diff --git a/openhands-sdk/openhands/sdk/utils/files.py b/openhands-sdk/openhands/sdk/utils/files.py index d35f72cf91d..9030125e0b8 100644 --- a/openhands-sdk/openhands/sdk/utils/files.py +++ b/openhands-sdk/openhands/sdk/utils/files.py @@ -1,6 +1,7 @@ import os import tempfile import time +from collections.abc import Callable from pathlib import Path from typing import Final @@ -9,13 +10,22 @@ # to ``...tmp`` next to ``path`` and then renames it to ````. _TEMP_SUFFIX: Final[str] = ".tmp" -# On Windows, ``os.replace`` raises ``PermissionError`` while another handle -# has the destination open, because Python opens files without delete -# sharing. Readers hold the file only while reading it, so the replace is -# retried after these delays (about 0.9 s in total) before the error is raised. -# On other platforms a replace never fails that way, and it is not retried. -_RETRY_REPLACE_ON_PERMISSION_ERROR: Final[bool] = os.name == "nt" -_REPLACE_RETRY_DELAYS: Final[tuple[float, ...]] = (0.01, 0.02, 0.05, 0.1, 0.2, 0.5) +# On Windows, a file being replaced cannot be opened by anyone else for an +# instant: ``os.replace`` holds the destination open with delete access, and +# Python opens files without delete sharing. So a replace fails while a reader +# has the destination open, and a read fails while a replace is in progress. +# Both raise ``PermissionError``, and both are retried after these delays +# (about 0.9 s in total) before the error is raised. On other platforms +# neither fails that way, and neither is retried. +_RETRY_ON_PERMISSION_ERROR: Final[bool] = os.name == "nt" +_PERMISSION_ERROR_RETRY_DELAYS: Final[tuple[float, ...]] = ( + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.5, +) def is_temp_file(path: Path) -> bool: @@ -27,16 +37,24 @@ def is_temp_file(path: Path) -> bool: return path.suffix == _TEMP_SUFFIX -def _replace(source: Path, destination: Path) -> None: - """``os.replace``, retried while a reader has ``destination`` open.""" - if _RETRY_REPLACE_ON_PERMISSION_ERROR: - for delay in _REPLACE_RETRY_DELAYS: +def _retry_on_permission_error[T](operation: Callable[[], T]) -> T: + """Run ``operation``, retried while a concurrent replace or read blocks it.""" + if _RETRY_ON_PERMISSION_ERROR: + for delay in _PERMISSION_ERROR_RETRY_DELAYS: try: - os.replace(source, destination) - return + return operation() except PermissionError: time.sleep(delay) - os.replace(source, destination) + return operation() + + +def read_text_retrying(path: Path) -> str: + """Read ``path`` as UTF-8, retried while a replace is in progress. + + For reading, without a lock, a file that ``atomic_write_text`` may replace + at any moment. + """ + return _retry_on_permission_error(lambda: path.read_text(encoding="utf-8")) def atomic_write_text(path: Path, value: str, mode: int = 0o600) -> None: @@ -57,7 +75,7 @@ def atomic_write_text(path: Path, value: str, mode: int = 0o600) -> None: file.write(value) file.flush() os.fsync(file.fileno()) - _replace(temporary_path, path) + _retry_on_permission_error(lambda: os.replace(temporary_path, path)) except BaseException: if fd >= 0: try: diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py index ccb217c557e..7fd33c96e7b 100644 --- a/tests/sdk/extensions/installation/test_installation_concurrency.py +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -148,14 +148,15 @@ def list_once_writer_is_loading() -> None: assert entry["enabled"] is (operation == "install") -def test_reader_thread_never_sees_partial_metadata_during_saves( +def test_reader_thread_finds_the_extension_during_saves( manager: InstallationManager[MockExtension], source_dir: Path, ): - """Torn read: ``get()`` takes no lock, so while another thread saves it - must still see a complete file, never a truncated one (which loads as - empty). On Windows this also covers the writer: a save whose replace - meets a reader's open handle must retry instead of raising. + """``get()`` takes no lock, so while another thread saves it must still + find the extension: never a truncated file (which loads as empty). On + Windows this also covers both sides of the replace: a save that meets the + reader's open handle, and a read that meets the save's replace, must each + retry instead of failing. """ manager.install(str(source_dir)) saving = threading.Event() @@ -178,10 +179,13 @@ def read_until_done() -> None: errors = _run_threads(("writer", save_repeatedly), ("reader", read_until_done)) - assert errors == [] + missed = [info for info in reads if info is None or info.source != str(source_dir)] + thread_errors = [f"{type(e).__name__}: {e}" for e in errors] + assert (thread_errors, len(missed)) == ([], 0), ( + f"thread errors: {thread_errors}; " + f"{len(missed)} of {len(reads)} reads did not find the extension" + ) assert reads, "the reader never ran while the writer was saving" - torn = [info for info in reads if info is None or info.source != str(source_dir)] - assert torn == [], f"{len(torn)} of {len(reads)} reads saw partial metadata" def test_get_does_not_wait_for_an_open_session( diff --git a/tests/sdk/utils/test_files.py b/tests/sdk/utils/test_files.py index 041be70927d..1829482e244 100644 --- a/tests/sdk/utils/test_files.py +++ b/tests/sdk/utils/test_files.py @@ -4,7 +4,11 @@ import pytest from openhands.sdk.utils import files -from openhands.sdk.utils.files import atomic_write_text, is_temp_file +from openhands.sdk.utils.files import ( + atomic_write_text, + is_temp_file, + read_text_retrying, +) def test_atomic_write_text_temp_file_is_marked_as_a_temp_file( @@ -76,7 +80,7 @@ def replace(src, dst): return replace -@pytest.mark.parametrize("failures", [1, len(files._REPLACE_RETRY_DELAYS)]) +@pytest.mark.parametrize("failures", [1, len(files._PERMISSION_ERROR_RETRY_DELAYS)]) def test_atomic_write_text_retries_replace_blocked_by_an_open_handle( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failures: int ) -> None: @@ -85,7 +89,7 @@ def test_atomic_write_text_retries_replace_blocked_by_an_open_handle( path.write_text("old") calls: list[Path] = [] sleeps: list[float] = [] - monkeypatch.setattr(files, "_RETRY_REPLACE_ON_PERMISSION_ERROR", True) + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", True) monkeypatch.setattr(files.time, "sleep", sleeps.append) monkeypatch.setattr( files.os, "replace", _replace_failing(failures, PermissionError(13, "x"), calls) @@ -95,7 +99,7 @@ def test_atomic_write_text_retries_replace_blocked_by_an_open_handle( assert path.read_text() == "new" assert len(calls) == failures + 1 - assert sleeps == list(files._REPLACE_RETRY_DELAYS[:failures]) + assert sleeps == list(files._PERMISSION_ERROR_RETRY_DELAYS[:failures]) assert [p.name for p in tmp_path.iterdir()] == ["data.json"] @@ -109,7 +113,7 @@ def test_atomic_write_text_raises_when_replace_stays_blocked( path.write_text("old") calls: list[Path] = [] sleeps: list[float] = [] - monkeypatch.setattr(files, "_RETRY_REPLACE_ON_PERMISSION_ERROR", True) + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", True) monkeypatch.setattr(files.time, "sleep", sleeps.append) monkeypatch.setattr( files.os, "replace", _replace_failing(100, PermissionError(13, "x"), calls) @@ -118,8 +122,8 @@ def test_atomic_write_text_raises_when_replace_stays_blocked( with pytest.raises(PermissionError): atomic_write_text(path, "new") - assert len(calls) == len(files._REPLACE_RETRY_DELAYS) + 1 - assert sleeps == list(files._REPLACE_RETRY_DELAYS) + assert len(calls) == len(files._PERMISSION_ERROR_RETRY_DELAYS) + 1 + assert sleeps == list(files._PERMISSION_ERROR_RETRY_DELAYS) assert path.read_text() == "old" assert [p.name for p in tmp_path.iterdir()] == ["data.json"] @@ -144,7 +148,7 @@ def test_atomic_write_text_does_not_retry_other_replace_failures( path = tmp_path / "data.json" calls: list[Path] = [] sleeps: list[float] = [] - monkeypatch.setattr(files, "_RETRY_REPLACE_ON_PERMISSION_ERROR", retry_enabled) + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", retry_enabled) monkeypatch.setattr(files.time, "sleep", sleeps.append) monkeypatch.setattr(files.os, "replace", _replace_failing(1, error, calls)) @@ -154,3 +158,51 @@ def test_atomic_write_text_does_not_retry_other_replace_failures( assert len(calls) == 1 assert sleeps == [] assert list(tmp_path.iterdir()) == [] + + +_MAX_RETRIES = len(files._PERMISSION_ERROR_RETRY_DELAYS) + + +@pytest.mark.parametrize( + ("retry_enabled", "failures", "reads", "succeeds"), + [ + (True, 1, 2, True), + (True, _MAX_RETRIES, _MAX_RETRIES + 1, True), + (True, _MAX_RETRIES + 1, _MAX_RETRIES + 1, False), + (False, 1, 1, False), + ], + ids=["one_failure", "max_failures", "too_many_failures", "off_windows"], +) +def test_read_text_retrying_retries_a_read_blocked_by_a_replace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + retry_enabled: bool, + failures: int, + reads: int, + succeeds: bool, +) -> None: + """On Windows, a read that meets a concurrent replace fails with + ``PermissionError`` and is retried, up to the last delay. + """ + path = tmp_path / "data.json" + path.write_text("content", encoding="utf-8") + calls: list[Path] = [] + real_read_text = Path.read_text + + def read_text(self: Path, *args, **kwargs) -> str: + calls.append(self) + if len(calls) <= failures: + raise PermissionError(13, "Permission denied") + return real_read_text(self, *args, **kwargs) + + monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", retry_enabled) + monkeypatch.setattr(files.time, "sleep", lambda delay: None) + monkeypatch.setattr(Path, "read_text", read_text) + + if succeeds: + assert read_text_retrying(path) == "content" + else: + with pytest.raises(PermissionError): + read_text_retrying(path) + + assert calls == [path] * reads From 47dc234d98bf0b8c7e7e2ae6da861a0a6e5c5411 Mon Sep 17 00:00:00 2001 From: adithya Date: Tue, 10 Nov 2026 18:28:33 +0530 Subject: [PATCH 07/15] fix(sdk): stage installs and swap them in under the metadata lock --- .../canvas_extensions/installed.py | 91 +++++--- .../sdk/extensions/installation/__init__.py | 6 +- .../sdk/extensions/installation/manager.py | 103 +++++++-- .../test_canvas_extensions_installed.py | 80 ++++++- .../test_canvas_extensions_refresh.py | 96 +++++++- tests/agent_server/test_plugins_router.py | 38 ++++ .../test_installation_concurrency.py | 215 ++++++++++++++++++ 7 files changed, 578 insertions(+), 51 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py index c41098d4438..c1bd68338af 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py @@ -9,8 +9,8 @@ forces ``enabled=False`` as an explicit post-write step. Staged refresh -- ``check``/``apply`` replace ``update()``/ -``install(force=True)``, which rmtree+copytree straight onto the live -install path with no staging directory. ``check`` fetches and validates +``install(force=True)``, which fetch and swap in one step, with nothing to +confirm in between. ``check`` fetches and validates into ``.staging/`` without touching the active install; ``apply`` swaps it in via two atomic renames (POSIX can't atomically replace a non-empty directory in one rename), rolling back if the second fails. @@ -18,6 +18,7 @@ import os import shutil +import tempfile from contextlib import suppress from pathlib import Path @@ -37,9 +38,12 @@ InstallationMetadata, ) from openhands.sdk.extensions.installation.manager import DEFAULT_CACHE_DIR +from openhands.sdk.logger import get_logger from openhands.sdk.utils.path import get_user_persistence_dir +logger = get_logger(__name__) + # Matches the InstalledPluginInfo naming convention. InstalledCanvasExtensionInfo = InstallationInfo @@ -132,9 +136,14 @@ def install_canvas_extension( """ installed_dir = _resolve_installed_dir(installed_dir) manager = _manager(installed_dir) - pre_existing = _tracked_names(installed_dir) - info = manager.install(source, ref=ref, repo_path=repo_path, force=force) - return _force_disable_new(manager, info, pre_existing) + # Whether this replaced an installed extension is decided inside the + # install's own metadata session: a snapshot taken before the install + # can be made stale by a concurrent uninstall. + outcome = manager.install_with_outcome( + source, ref=ref, repo_path=repo_path, force=force + ) + pre_existing = {outcome.info.name} if outcome.was_installed else set() + return _force_disable_new(manager, outcome.info, pre_existing) def uninstall_canvas_extension(name: str, installed_dir: Path | None = None) -> bool: @@ -422,8 +431,7 @@ def apply_canvas_extension_update( """ installed_dir = _resolve_installed_dir(installed_dir) manager = _manager(installed_dir) - current_info = manager.get(name) - if current_info is None: + if manager.get(name) is None: return None staged_path = _staged_path(installed_dir, name, resolved_ref) @@ -435,31 +443,50 @@ def apply_canvas_extension_update( manifest = _load_validated_manifest(staged_path, name) install_path = installed_dir / name - backup_path = _staging_root(installed_dir) / f"{name}.previous" - shutil.rmtree(backup_path, ignore_errors=True) - - # POSIX rename() can't atomically replace a non-empty directory, so the - # active bundle moves aside first; roll back if the second rename fails. - os.replace(install_path, backup_path) - try: - os.replace(staged_path, install_path) - except OSError: - os.replace(backup_path, install_path) - raise - shutil.rmtree(backup_path, ignore_errors=True) - shutil.rmtree(_staging_root(installed_dir) / name, ignore_errors=True) - - info = InstallationInfo.from_extension( - manifest, - source=current_info.source, - install_path=install_path, - requested_ref=current_info.requested_ref, - resolved_ref=resolved_ref, - repo_path=current_info.repo_path, + # A directory of its own per apply: the previous bundle is deleted after + # the lock is released, so a shared backup path could be cleared while + # another apply is between its renames. + backup_root = Path( + tempfile.mkdtemp(prefix=f"{name}.previous-", dir=_staging_root(installed_dir)) ) - info.enabled = enabled - - with manager.metadata_session as session: - session.extensions[name] = info + backup_path = backup_root / name + try: + # The swap and the metadata write share one session, so a concurrent + # list can't see the install path missing between the two renames + # and prune the entry. + with manager.metadata_session as session: + current_info = session.extensions.get(name) + if current_info is None or not install_path.is_dir(): + return None + + # POSIX rename() can't atomically replace a non-empty directory, so + # the active bundle moves aside first; roll back if the second + # rename fails. + os.replace(install_path, backup_path) + try: + os.replace(staged_path, install_path) + except OSError: + os.replace(backup_path, install_path) + raise + + info = InstallationInfo.from_extension( + manifest, + source=current_info.source, + install_path=install_path, + requested_ref=current_info.requested_ref, + resolved_ref=resolved_ref, + repo_path=current_info.repo_path, + ) + info.enabled = enabled + session.extensions[name] = info + finally: + if backup_path.exists() and not install_path.exists(): + logger.error( + f"Could not restore {install_path} after a failed update;" + f" the previous version was kept at {backup_path}" + ) + else: + shutil.rmtree(backup_root, ignore_errors=True) + shutil.rmtree(_staging_root(installed_dir) / name, ignore_errors=True) return info diff --git a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py index 5e8aba19905..b5626467313 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py @@ -3,7 +3,10 @@ ExtensionProtocol, InstallationInterface, ) -from openhands.sdk.extensions.installation.manager import InstallationManager +from openhands.sdk.extensions.installation.manager import ( + InstallationManager, + InstallationOutcome, +) from openhands.sdk.extensions.installation.metadata import ( InstallationMetadata, InstallationMetadataLoadError, @@ -18,5 +21,6 @@ "InstallationManager", "InstallationMetadata", "InstallationMetadataLoadError", + "InstallationOutcome", "MetadataSession", ] diff --git a/openhands-sdk/openhands/sdk/extensions/installation/manager.py b/openhands-sdk/openhands/sdk/extensions/installation/manager.py index a6c54693587..9c486f1c5ef 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/manager.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/manager.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import shutil import tempfile from dataclasses import dataclass @@ -31,6 +32,35 @@ DEFAULT_CACHE_DIR = get_user_persistence_dir() / "cache" / "extensions" +@dataclass(frozen=True) +class InstallationOutcome: + """Result of ``InstallationManager.install_with_outcome``.""" + + info: InstallationInfo + was_installed: bool + """Whether an extension of this name was installed (tracked, with its + directory present) when this install replaced it. Decided inside the + metadata session that records the install.""" + + +def _swap_directory(staged_path: Path, install_path: Path, backup_path: Path) -> None: + """Move ``staged_path`` to ``install_path``. + + POSIX ``rename`` can't replace a non-empty directory, so an existing + ``install_path`` is first moved to ``backup_path``. If the second rename + fails, the first is undone and the error is re-raised. + """ + replacing = install_path.exists() + if replacing: + os.replace(install_path, backup_path) + try: + os.replace(staged_path, install_path) + except OSError: + if replacing: + os.replace(backup_path, install_path) + raise + + @dataclass class InstallationManager[T: ExtensionProtocol]: """Generic manager for installing, tracking, and loading extensions. @@ -88,6 +118,22 @@ def install( FileExistsError: If extension is already installed and force=False. ValueError: If the extension name is invalid. """ + return self._install(source, ref=ref, repo_path=repo_path, force=force).info + + def install_with_outcome( + self, + source: str | Path, + ref: str | None = None, + repo_path: str | None = None, + force: bool = False, + ) -> InstallationOutcome: + """Install as ``install`` does, and report whether it replaced an + installed extension. + + For callers whose handling of the result depends on that. Checking + with ``get()`` before calling ``install`` can be out of date by the + time the install is recorded; this is decided while it is recorded. + """ return self._install(source, ref=ref, repo_path=repo_path, force=force) def _install( @@ -97,7 +143,7 @@ def _install( repo_path: str | None = None, force: bool = False, root_name: str | None = None, - ) -> InstallationInfo: + ) -> InstallationOutcome: """Install as ``install`` does; ``root_name`` names a repository root. ``root_name`` replaces the repository's name as the directory name a @@ -127,14 +173,6 @@ def _install( f" at {install_path}. Use force=True to overwrite." ) - if install_path.exists(): - logger.info(f"Removing existing installation of '{extension.name}'") - shutil.rmtree(install_path) - - logger.info(f"Installing extension '{extension.name}' to {install_path}") - self.installation_dir.mkdir(parents=True, exist_ok=True) - shutil.copytree(fetched_path, install_path) - info = InstallationInfo.from_extension( extension, source=source, @@ -144,16 +182,49 @@ def _install( repo_path=repo_path, ) - with self.metadata_session as session: - existing = session.extensions.get(extension.name) - if existing is not None: - info.enabled = existing.enabled - session.extensions[extension.name] = info + # Copy outside the lock into a dot-prefixed directory, which discovery + # skips: a concurrent list never sees a partly copied extension, and + # lock time doesn't grow with its size. Only the swap into place and + # the metadata write happen under the lock. Staging inside the + # installation directory keeps the swap a same-filesystem rename. + logger.info(f"Installing extension '{extension.name}' to {install_path}") + self.installation_dir.mkdir(parents=True, exist_ok=True) + staging_dir = Path( + tempfile.mkdtemp( + prefix=f".{extension.name}.installing-", dir=self.installation_dir + ) + ) + backup_path = staging_dir / "previous" + try: + staged_path = staging_dir / "new" + shutil.copytree(fetched_path, staged_path) + + with self.metadata_session as session: + if install_path.exists() and not force: + raise FileExistsError( + f"Extension '{extension.name}' is already installed" + f" at {install_path}. Use force=True to overwrite." + ) + existing = session.extensions.get(extension.name) + was_installed = existing is not None and install_path.exists() + if existing is not None: + info.enabled = existing.enabled + _swap_directory(staged_path, install_path, backup_path) + session.extensions[extension.name] = info + finally: + # The previous version is deleted after the lock is released. + if backup_path.exists() and not install_path.exists(): + logger.error( + f"Could not restore {install_path} after a failed install;" + f" the previous version was kept at {backup_path}" + ) + else: + shutil.rmtree(staging_dir, ignore_errors=True) logger.info( f"Successfully installed extension '{extension.name}' v{info.version}" ) - return info + return InstallationOutcome(info=info, was_installed=was_installed) def _load_fetched( self, @@ -387,4 +458,4 @@ def update(self, name: str) -> InstallationInfo | None: repo_path=current_info.repo_path, force=True, root_name=name, - ) + ).info diff --git a/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py b/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py index 4703528e9ac..65089d08da7 100644 --- a/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py +++ b/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py @@ -8,6 +8,8 @@ import inspect import json +import shutil +import threading from pathlib import Path from typing import Any @@ -25,7 +27,10 @@ uninstall_canvas_extension, ) from openhands.agent_server.canvas_extensions.manifest import MANIFEST_FILENAME -from openhands.sdk.extensions.installation import InstallationMetadata +from openhands.sdk.extensions.installation import ( + InstallationMetadata, + manager as installation_manager, +) from .conftest import write_extension as _write_extension @@ -397,3 +402,76 @@ def test_disable_nonexistent_extension_returns_false(installed_dir: Path): def test_enable_nonexistent_extension_returns_false(installed_dir: Path): assert enable_canvas_extension("nonexistent", installed_dir=installed_dir) is False + + +# ============================================================================ +# Force reinstall racing other operations +# ============================================================================ + + +def test_force_reinstall_with_a_concurrent_list_keeps_disabled_state( + extension_dir: Path, installed_dir: Path, monkeypatch: pytest.MonkeyPatch +): + """A list call while a force reinstall copies the new bundle must not + prune the disabled entry: the reinstall would then record it as new and + enabled, skipping the approval that enabling requires. + """ + install_canvas_extension(str(extension_dir), installed_dir=installed_dir) + real_copytree = shutil.copytree + listed: list[list[Any]] = [] + + def copytree(src, dst, *args, **kwargs): + if Path(dst).is_relative_to(installed_dir.resolve()) and not listed: + thread = threading.Thread( + target=lambda: listed.append( + list_installed_canvas_extensions(installed_dir=installed_dir) + ) + ) + thread.start() + thread.join(10) + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(installation_manager.shutil, "copytree", copytree) + + info = install_canvas_extension( + str(extension_dir), installed_dir=installed_dir, force=True + ) + + assert [(i.name, i.enabled) for i in listed[0]] == [("my-extension", False)] + assert info.enabled is False + on_disk = get_installed_canvas_extension( + "my-extension", installed_dir=installed_dir + ) + assert on_disk is not None + assert on_disk.enabled is False + + +def test_force_reinstall_after_a_concurrent_uninstall_lands_disabled( + extension_dir: Path, installed_dir: Path, monkeypatch: pytest.MonkeyPatch +): + """Whether a force install replaced an installed extension is decided when + the install is recorded. Decided earlier, an uninstall in between would + let the new entry pass as a reinstall and land enabled. + """ + install_canvas_extension(str(extension_dir), installed_dir=installed_dir) + enable_canvas_extension("my-extension", installed_dir=installed_dir) + real_copytree = shutil.copytree + uninstalled: list[bool] = [] + + def copytree(src, dst, *args, **kwargs): + if not uninstalled: + uninstalled.append( + uninstall_canvas_extension("my-extension", installed_dir=installed_dir) + ) + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(installation_manager.shutil, "copytree", copytree) + + info = install_canvas_extension( + str(extension_dir), installed_dir=installed_dir, force=True + ) + + assert uninstalled == [True] + assert info.enabled is False + on_disk = InstallationMetadata.load_from_dir(installed_dir) + assert on_disk.extensions["my-extension"].enabled is False diff --git a/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py b/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py index 19fd8785f6d..4ea58fe833c 100644 --- a/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py +++ b/tests/agent_server/canvas_extensions/test_canvas_extensions_refresh.py @@ -2,6 +2,7 @@ import json import os +import threading from pathlib import Path from unittest.mock import patch @@ -15,10 +16,12 @@ enable_canvas_extension, get_installed_canvas_extension, install_canvas_extension, + list_installed_canvas_extensions, uninstall_canvas_extension, ) from openhands.agent_server.canvas_extensions.manifest import MANIFEST_FILENAME from openhands.sdk.extensions.fetch import ExtensionFetchError +from openhands.sdk.extensions.installation import InstallationMetadata from .conftest import write_extension as _write_extension @@ -523,7 +526,7 @@ def test_apply_cleans_up_staging_after_success(extension_dir: Path, installed: P ) assert not (installed / ".staging" / "my-extension").exists() - assert not (installed / ".staging" / "my-extension.previous").exists() + assert list((installed / ".staging").glob("my-extension.previous*")) == [] def test_double_apply_without_recheck_raises(extension_dir: Path, installed: Path): @@ -674,6 +677,97 @@ def test_apply_leaves_no_backup_when_first_rename_fails( assert staged.is_dir() +def test_apply_keeps_the_previous_bundle_when_the_rollback_fails( + extension_dir: Path, installed: Path +): + """If the rollback rename fails too, the previous bundle is the only copy + left, so it stays in its backup directory instead of being deleted. + """ + _write_extension(extension_dir, version="2.0.0") + check = check_canvas_extension_update("my-extension", installed_dir=installed) + assert check is not None + install_path = installed / "my-extension" + real_replace = os.replace + renames = {"n": 0} + + def flaky_replace(src, dst): + if install_path in (Path(src), Path(dst)): + renames["n"] += 1 + if renames["n"] in (2, 3): + raise OSError("simulated rename failure") + return real_replace(src, dst) + + with patch("os.replace", side_effect=flaky_replace): + with pytest.raises(OSError, match="simulated rename failure"): + apply_canvas_extension_update( + "my-extension", + resolved_ref=check.resolved_ref, + enabled=True, + installed_dir=installed, + ) + + assert not install_path.exists() + [backup_root] = (installed / ".staging").glob("my-extension.previous-*") + assert _active_version(backup_root) == "1.0.0" + + +def test_list_cannot_run_between_the_apply_renames( + extension_dir: Path, installed: Path, monkeypatch: pytest.MonkeyPatch +): + """The install path is missing between apply's two renames. The metadata + lock must keep a list call from loading in that window, where it would + prune the entry. + """ + _write_extension(extension_dir, version="2.0.0") + check = check_canvas_extension_update("my-extension", installed_dir=installed) + assert check is not None + install_path = installed / "my-extension" + original_load = InstallationMetadata._load_for_session.__func__ + list_loaded = threading.Event() + + def load(cls: type[InstallationMetadata], directory: Path): + result = original_load(cls, directory) + if threading.current_thread().name == "list": + list_loaded.set() + return result + + monkeypatch.setattr(InstallationMetadata, "_load_for_session", classmethod(load)) + real_replace = os.replace + lists: list[threading.Thread] = [] + listed: list[list] = [] + + def replace(src, dst): + real_replace(src, dst) + if Path(src) == install_path and not lists: + thread = threading.Thread( + target=lambda: listed.append( + list_installed_canvas_extensions(installed_dir=installed) + ), + name="list", + daemon=True, + ) + lists.append(thread) + thread.start() + assert not list_loaded.wait(0.5), ( + "list loaded metadata between the apply renames" + ) + + monkeypatch.setattr(os, "replace", replace) + + apply_canvas_extension_update( + "my-extension", + resolved_ref=check.resolved_ref, + enabled=False, + installed_dir=installed, + ) + + assert len(lists) == 1, "apply did not move the active bundle aside" + lists[0].join(10) + assert [(i.name, i.version, i.enabled) for i in listed[0]] == [ + ("my-extension", "2.0.0", False) + ] + + def test_apply_rejects_manifest_that_became_invalid_since_check( extension_dir: Path, installed: Path ): diff --git a/tests/agent_server/test_plugins_router.py b/tests/agent_server/test_plugins_router.py index 59b3cbb5db0..1e2c432d293 100644 --- a/tests/agent_server/test_plugins_router.py +++ b/tests/agent_server/test_plugins_router.py @@ -7,6 +7,7 @@ import json import shutil import subprocess +import threading from pathlib import Path from unittest.mock import patch @@ -126,6 +127,43 @@ def test_refresh_returns_updated_plugin(client: TestClient, tmp_path: Path): assert refreshed.json()["plugin"]["name"] == "demo-plugin" +def test_refresh_with_a_concurrent_list_keeps_a_disabled_plugin_disabled( + client: TestClient, tmp_path: Path, monkeypatch +): + """#5985 through the API: a GET /plugins/installed while a refresh copies + the new version must not prune the plugin's entry, which would make the + refresh re-enable a disabled plugin. + """ + src = _make_installable_plugin(tmp_path / "src", "demo-plugin") + client.post("/plugins/install", json={"source": str(src)}) + client.patch("/plugins/installed/demo-plugin", json={"enabled": False}) + store = (tmp_path / "installed-store").resolve() + real_copytree = shutil.copytree + listed = [] + + def copytree(source, destination, *args, **kwargs): + if Path(destination).is_relative_to(store) and not listed: + thread = threading.Thread( + target=lambda: listed.append(client.get("/plugins/installed")) + ) + thread.start() + thread.join(10) + return real_copytree(source, destination, *args, **kwargs) + + monkeypatch.setattr(shutil, "copytree", copytree) + + refreshed = client.post("/plugins/installed/demo-plugin/refresh") + + assert refreshed.status_code == 200 + assert len(listed) == 1, "the refresh did not copy into the install store" + assert listed[0].status_code == 200 + assert [(p["name"], p["enabled"]) for p in listed[0].json()["plugins"]] == [ + ("demo-plugin", False) + ] + plugin = client.get("/plugins/installed/demo-plugin").json() + assert (plugin["enabled"], plugin["source"]) == (False, str(src)) + + def test_refresh_vanished_source_is_400_and_keeps_plugin( client: TestClient, tmp_path: Path ): diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py index 7fd33c96e7b..cbbacb0fb3c 100644 --- a/tests/sdk/extensions/installation/test_installation_concurrency.py +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -1,6 +1,8 @@ """Concurrency regression tests for installation metadata""" import json +import os +import shutil import threading from collections.abc import Callable from pathlib import Path @@ -14,6 +16,7 @@ InstallationManager, InstallationMetadata, MetadataSession, + manager as manager_module, metadata as metadata_module, ) @@ -273,3 +276,215 @@ def test_lock_is_released_when_a_session_fails( assert errors == [] assert excinfo.value is not None assert len(sessions) == (0 if failure == "load_fails" else 1) + + +# ============================================================================ +# Directory swap during install(force=True) and update() +# ============================================================================ + + +def _staging_dirs(manager: InstallationManager[MockExtension]) -> list[Path]: + return list(manager.installation_dir.glob(".magic-test.installing-*")) + + +def _install_disabled_with_new_version( + manager: InstallationManager[MockExtension], source_dir: Path +) -> Path: + """Install and disable magic-test, then add a file only the new version has.""" + manager.install(str(source_dir)) + manager.disable("magic-test") + (source_dir / "new-version").write_text("2") + return manager.installation_dir / "magic-test" + + +def test_list_while_update_copies_keeps_the_entry( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """#5985: a list call while update() copies the new version must not see + the extension missing (and prune its entry) or half copied (and rediscover + it as local), so a disabled extension stays disabled. + """ + _install_disabled_with_new_version(manager, source_dir) + real_copytree = shutil.copytree + listed: list[list[InstallationInfo]] = [] + + def copytree(src, dst, *args, **kwargs): + if Path(dst).is_relative_to(manager.installation_dir) and not listed: + errors = _run_threads( + ("list", lambda: listed.append(manager.list_installed())) + ) + assert errors == [] + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(manager_module.shutil, "copytree", copytree) + + manager.update("magic-test") + + assert len(listed) == 1, "update() did not copy into the installation directory" + assert [(i.name, i.source, i.enabled) for i in listed[0]] == [ + ("magic-test", str(source_dir), False) + ] + info = manager.get("magic-test") + assert info is not None + assert (info.source, info.enabled) == (str(source_dir), False) + assert (manager.installation_dir / "magic-test" / "new-version").exists() + assert _staging_dirs(manager) == [] + + +def test_list_cannot_run_between_the_swap_renames( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """The swap leaves the install path missing between its two renames. The + metadata lock must keep a list call from loading in that window, where it + would prune the entry. + """ + install_path = _install_disabled_with_new_version(manager, source_dir) + original_load = InstallationMetadata._load_for_session.__func__ + list_loaded = threading.Event() + + def load(cls: type[InstallationMetadata], directory: Path): + result = original_load(cls, directory) + if threading.current_thread().name == "list": + list_loaded.set() + return result + + monkeypatch.setattr(InstallationMetadata, "_load_for_session", classmethod(load)) + real_replace = os.replace + lists: list[threading.Thread] = [] + listed: list[list[InstallationInfo]] = [] + + def replace(src, dst) -> None: + real_replace(src, dst) + if Path(src) == install_path and not lists: + thread = threading.Thread( + target=lambda: listed.append(manager.list_installed()), + name="list", + daemon=True, + ) + lists.append(thread) + thread.start() + assert not list_loaded.wait(0.5), ( + "list_installed loaded metadata between the swap renames" + ) + + monkeypatch.setattr(manager_module.os, "replace", replace) + + manager.update("magic-test") + + assert len(lists) == 1, "update() did not move the installed directory aside" + lists[0].join(10) + assert [(i.name, i.enabled) for i in listed[0]] == [("magic-test", False)] + + +@pytest.mark.parametrize( + "failing_renames", + [{1}, {2}, {2, 3}], + ids=["first_rename", "second_rename", "second_rename_and_rollback"], +) +def test_failed_swap_keeps_the_installed_version( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, + failing_renames: set[int], +): + """A rename that fails during the swap (on Windows: a file in the + directory is open) must leave the previous version installed and its + metadata unchanged. If even the rollback fails, the previous version is + kept in the staging directory rather than deleted. + """ + install_path = _install_disabled_with_new_version(manager, source_dir) + real_replace = os.replace + renames: list[tuple[Path, Path]] = [] + + def replace(src, dst) -> None: + if install_path in (Path(src), Path(dst)): + renames.append((Path(src), Path(dst))) + if len(renames) in failing_renames: + raise PermissionError(13, "simulated rename failure") + real_replace(src, dst) + + monkeypatch.setattr(manager_module.os, "replace", replace) + + with pytest.raises(PermissionError, match="simulated rename failure"): + manager.update("magic-test") + + monkeypatch.setattr(manager_module.os, "replace", real_replace) + info = InstallationMetadata.load_from_dir(manager.installation_dir).extensions[ + "magic-test" + ] + assert (info.source, info.enabled) == (str(source_dir), False) + if failing_renames == {2, 3}: + assert not install_path.exists() + [staging_dir] = _staging_dirs(manager) + assert (staging_dir / "previous" / "extension.json").exists() + assert not (staging_dir / "previous" / "new-version").exists() + else: + assert (install_path / "extension.json").exists() + assert not (install_path / "new-version").exists() + assert _staging_dirs(manager) == [] + + +def test_previous_version_is_deleted_after_the_lock_is_released( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Deleting the previous version takes time proportional to its size, so + it must happen outside the lock: another session can open meanwhile. + """ + _install_disabled_with_new_version(manager, source_dir) + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + real_rmtree = shutil.rmtree + probes: list[list[BaseException]] = [] + + def rmtree(path, *args, **kwargs): + if Path(path).name.startswith(".magic-test.installing-"): + probes.append( + _run_threads( + ( + "probe", + lambda: InstallationMetadata.open( + manager.installation_dir + ).__exit__(None, None, None), + ) + ) + ) + return real_rmtree(path, *args, **kwargs) + + monkeypatch.setattr(manager_module.shutil, "rmtree", rmtree) + + manager.update("magic-test") + + assert probes == [[]] + + +def test_install_does_not_replace_a_directory_that_appeared_meanwhile( + manager: InstallationManager[MockExtension], + source_dir: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Without force, an install that finds the extension installed by the + time it takes the lock (another install finished first) must fail, not + swap the other install's directory away. + """ + install_path = manager.installation_dir / "magic-test" + real_copytree = shutil.copytree + + def copytree(src, dst, *args, **kwargs): + if not install_path.exists(): + install_path.mkdir(parents=True) + (install_path / "winner").write_text("1") + return real_copytree(src, dst, *args, **kwargs) + + monkeypatch.setattr(manager_module.shutil, "copytree", copytree) + + with pytest.raises(FileExistsError): + manager.install(str(source_dir)) + + assert (install_path / "winner").exists() + assert not (install_path / "extension.json").exists() + assert _staging_dirs(manager) == [] From 0b9f406dbddebca41d069b6bc5f30fc28333ce85 Mon Sep 17 00:00:00 2001 From: adithya Date: Tue, 10 Nov 2026 20:21:01 +0530 Subject: [PATCH 08/15] fix(sdk): retry a blocked metadata replace or read every 10 ms --- openhands-sdk/openhands/sdk/utils/files.py | 17 ++++++----------- tests/sdk/utils/test_files.py | 8 +++++++- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/openhands-sdk/openhands/sdk/utils/files.py b/openhands-sdk/openhands/sdk/utils/files.py index 9030125e0b8..4e3c0d1eeec 100644 --- a/openhands-sdk/openhands/sdk/utils/files.py +++ b/openhands-sdk/openhands/sdk/utils/files.py @@ -14,18 +14,13 @@ # instant: ``os.replace`` holds the destination open with delete access, and # Python opens files without delete sharing. So a replace fails while a reader # has the destination open, and a read fails while a replace is in progress. -# Both raise ``PermissionError``, and both are retried after these delays -# (about 0.9 s in total) before the error is raised. On other platforms -# neither fails that way, and neither is retried. +# Both raise ``PermissionError``, and both are retried every 10 ms for about +# 1 s before the error is raised. Readers hold the file only for one read, so +# under a steady stream of reads many short attempts get through where a few +# spaced-out ones can each land on a read. On other platforms neither fails +# that way, and neither is retried. _RETRY_ON_PERMISSION_ERROR: Final[bool] = os.name == "nt" -_PERMISSION_ERROR_RETRY_DELAYS: Final[tuple[float, ...]] = ( - 0.01, - 0.02, - 0.05, - 0.1, - 0.2, - 0.5, -) +_PERMISSION_ERROR_RETRY_DELAYS: Final[tuple[float, ...]] = (0.01,) * 100 def is_temp_file(path: Path) -> bool: diff --git a/tests/sdk/utils/test_files.py b/tests/sdk/utils/test_files.py index 1829482e244..fca2b33f78b 100644 --- a/tests/sdk/utils/test_files.py +++ b/tests/sdk/utils/test_files.py @@ -116,7 +116,13 @@ def test_atomic_write_text_raises_when_replace_stays_blocked( monkeypatch.setattr(files, "_RETRY_ON_PERMISSION_ERROR", True) monkeypatch.setattr(files.time, "sleep", sleeps.append) monkeypatch.setattr( - files.os, "replace", _replace_failing(100, PermissionError(13, "x"), calls) + files.os, + "replace", + _replace_failing( + len(files._PERMISSION_ERROR_RETRY_DELAYS) + 1, + PermissionError(13, "x"), + calls, + ), ) with pytest.raises(PermissionError): From e724f2ffde9a3139cc31442601d169a081357e4d Mon Sep 17 00:00:00 2001 From: adithya Date: Tue, 10 Nov 2026 21:58:35 +0530 Subject: [PATCH 09/15] fix(agent-server): run lock-taking Canvas calls off the event loop The enable/disable, uninstall and revoke-secret-grant Canvas endpoints are async def and called disable/enable/uninstall_canvas_extension directly. Those take the installation metadata FileLock, which waits with a blocking sleep for up to 30 s, so a held lock stalled the whole event loop. Run them through asyncio.to_thread, as the install endpoint already does. Adds a regression test that holds the metadata lock and checks /alive still answers while each endpoint waits for it. --- .../agent_server/canvas_extensions_router.py | 14 +- ...t_canvas_extensions_lock_responsiveness.py | 148 ++++++++++++++++++ 2 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 tests/agent_server/test_canvas_extensions_lock_responsiveness.py diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py b/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py index ed4068ee5c8..f39ef44dc81 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py @@ -315,7 +315,9 @@ async def set_canvas_extension_enabled_endpoint( async with lock: if not request.enabled: await _revoke_and_stop_backend(http_request, extension_name) - if not disable_canvas_extension(name=extension_name): + if not await asyncio.to_thread( + disable_canvas_extension, name=extension_name + ): raise HTTPException(404, "Canvas App is not installed") return UpdateCanvasExtensionStateResponse( name=extension_name, enabled=False @@ -363,13 +365,13 @@ async def set_canvas_extension_enabled_endpoint( ) except (ValueError, RuntimeError, OSError) as exc: await _revoke_and_stop_backend(http_request, extension_name) - disable_canvas_extension(name=extension_name) + await asyncio.to_thread(disable_canvas_extension, name=extension_name) raise HTTPException(409, str(exc)) from exc except BaseException: await _revoke_and_stop_backend(http_request, extension_name) - disable_canvas_extension(name=extension_name) + await asyncio.to_thread(disable_canvas_extension, name=extension_name) raise - if not enable_canvas_extension(name=extension_name): + if not await asyncio.to_thread(enable_canvas_extension, name=extension_name): raise HTTPException(404, "Canvas App is not installed") return UpdateCanvasExtensionStateResponse(name=extension_name, enabled=True) @@ -385,7 +387,7 @@ async def uninstall_canvas_extension_endpoint( ) -> UninstallCanvasExtensionResponse: """Uninstall a canvas extension by name while retaining backend data.""" await _revoke_and_stop_backend(request, extension_name) - if not uninstall_canvas_extension(name=extension_name): + if not await asyncio.to_thread(uninstall_canvas_extension, name=extension_name): raise HTTPException( status_code=404, detail=f"Canvas extension '{extension_name}' is not installed", @@ -576,6 +578,6 @@ async def revoke_canvas_extension_backend_secrets_endpoint( ) -> BackendStatus: """Revoke consent, terminate the process group and invalidate bridge sessions.""" async with _lifecycle_lock(request, extension_name): - disable_canvas_extension(name=extension_name) + await asyncio.to_thread(disable_canvas_extension, name=extension_name) await _backend_manager(request).revoke_secrets(extension_name) return await _revoke_and_stop_backend(request, extension_name) diff --git a/tests/agent_server/test_canvas_extensions_lock_responsiveness.py b/tests/agent_server/test_canvas_extensions_lock_responsiveness.py new file mode 100644 index 00000000000..a2e1aa05376 --- /dev/null +++ b/tests/agent_server/test_canvas_extensions_lock_responsiveness.py @@ -0,0 +1,148 @@ +"""A Canvas request waiting on the installation metadata lock must not stall +the server. + +``InstallationMetadata.open()`` takes a ``filelock.FileLock`` whose ``acquire`` +polls with a blocking sleep (up to 30 s). The ``async def`` Canvas endpoints +must run the lock-taking calls off the event loop, as the install endpoint +already does, so ``/alive`` keeps answering while another process holds the +lock. Same check as ``test_profile_store_lock_responsiveness.py``, except that +these handlers stop the backend before taking the lock, so the test waits for +the lock attempt itself instead of a fixed delay. +""" + +import asyncio +import threading +import time +from pathlib import Path + +import httpx +import pytest +from filelock import FileLock + +from openhands.agent_server.api import create_app +from openhands.agent_server.canvas_extensions.installed import ( + enable_canvas_extension, + install_canvas_extension, +) +from openhands.agent_server.config import Config +from openhands.agent_server.persistence import reset_stores +from openhands.sdk.extensions.installation import InstallationMetadata + +from .canvas_extensions.conftest import write_extension + + +# Upper bound for the held lock: if the handler blocks the event loop, nothing +# can release the lock from the test, so a timer frees it after this long. +_SAFETY_RELEASE_SECONDS = 5.0 +_ALIVE_BUDGET_SECONDS = 2.0 +_NAME = "demo-extension" + + +@pytest.mark.parametrize( + ("method", "path", "json_body", "enabled_before"), + [ + pytest.param( + "PATCH", + f"/api/canvas-extensions/installed/{_NAME}", + {"enabled": False}, + True, + id="disable", + ), + pytest.param( + "PATCH", + f"/api/canvas-extensions/installed/{_NAME}", + {"enabled": True}, + False, + id="enable", + ), + pytest.param( + "DELETE", + f"/api/canvas-extensions/installed/{_NAME}", + None, + False, + id="uninstall", + ), + pytest.param( + "DELETE", + f"/api/canvas-extensions/installed/{_NAME}/backend/secret-grant", + None, + True, + id="revoke-secret-grant", + ), + ], +) +async def test_alive_answers_while_canvas_request_waits_for_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + method: str, + path: str, + json_body: dict | None, + enabled_before: bool, +) -> None: + installed_dir = tmp_path / "canvas-installed" + monkeypatch.setattr( + "openhands.agent_server.canvas_extensions.installed." + "get_installed_canvas_extensions_dir", + lambda: installed_dir, + ) + monkeypatch.setenv("OH_PERSISTENCE_DIR", str(tmp_path)) + reset_stores() + install_canvas_extension( + source=str(write_extension(tmp_path / "src" / _NAME, name=_NAME)) + ) + if enabled_before: + assert enable_canvas_extension(_NAME) + app = create_app( + Config(static_files_path=None, session_api_keys=[], secret_key=None) + ) + + # Set from the thread that is about to wait on the lock. The time is taken + # there too: if the handler blocks the event loop, the test's own clock + # reading would only resume after the lock is released. + lock_requested = threading.Event() + lock_requested_at: list[float] = [] + acquire_lock = InstallationMetadata._acquire_lock + + def signalling_acquire_lock(installed_dir: Path): + lock_requested_at.append(time.monotonic()) + lock_requested.set() + return acquire_lock(installed_dir) + + monkeypatch.setattr( + InstallationMetadata, "_acquire_lock", staticmethod(signalling_acquire_lock) + ) + + # A separate FileLock instance stands in for another process holding the + # metadata lock. ``thread_local=False`` lets the safety timer release it. + held = FileLock(installed_dir / ".metadata.lock", thread_local=False) + held.acquire() + safety = threading.Timer(_SAFETY_RELEASE_SECONDS, held.release) + safety.start() + try: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + waiting = asyncio.create_task( + client.request(method, path, json=json_body, timeout=60) + ) + assert await asyncio.to_thread(lock_requested.wait, 10) + alive = await client.get("/alive", timeout=60) + elapsed = time.monotonic() - lock_requested_at[0] + + assert alive.status_code == 200 + assert elapsed < _ALIVE_BUDGET_SECONDS, ( + f"/alive took {elapsed:.1f}s while a Canvas request waited " + "for the metadata lock: the wait is blocking the event loop" + ) + # The Canvas request was really still waiting on the lock. + assert not waiting.done() + + safety.cancel() + held.release() + response = await asyncio.wait_for(waiting, timeout=30) + assert response.status_code == 200, response.text + finally: + safety.cancel() + if held.is_locked: + held.release(force=True) + reset_stores() From ebdf32b9480e41ed9da475f2bbf2bf376e7002ef Mon Sep 17 00:00:00 2001 From: adithya Date: Wed, 11 Nov 2026 16:28:16 +0530 Subject: [PATCH 10/15] fix(sdk): answer 503 when the installation metadata lock times out The metadata lock timeout raised a bare TimeoutError, an OSError: plugin and skill routes answered 500 and the Canvas install route reported it as an invalid extension (422). Raise a dedicated InstallationMetadataLockTimeoutError instead and map it to 503 in one app-level handler, next to the existing domain-error handlers, so every plugin, skill and Canvas route returns a retryable 503. InstallationMetadataLoadError still answers 500: retrying cannot fix a corrupt metadata file. --- .../openhands/agent_server/api.py | 20 +++ .../agent_server/canvas_extensions_router.py | 16 +- .../openhands/agent_server/plugins_router.py | 24 ++- .../openhands/agent_server/skills_router.py | 24 ++- .../sdk/extensions/installation/__init__.py | 2 + .../sdk/extensions/installation/metadata.py | 28 +++- .../test_installation_lock_timeout.py | 140 ++++++++++++++++++ .../test_installation_concurrency.py | 7 +- 8 files changed, 244 insertions(+), 17 deletions(-) create mode 100644 tests/agent_server/test_installation_lock_timeout.py diff --git a/openhands-agent-server/openhands/agent_server/api.py b/openhands-agent-server/openhands/agent_server/api.py index 59d66352edd..3a9e00ff78a 100644 --- a/openhands-agent-server/openhands/agent_server/api.py +++ b/openhands-agent-server/openhands/agent_server/api.py @@ -119,6 +119,9 @@ from openhands.agent_server.vscode_router import vscode_router from openhands.agent_server.vscode_service import get_vscode_service from openhands.agent_server.workspaces_router import workspaces_router +from openhands.sdk.extensions.installation import ( + InstallationMetadataLockTimeoutError, +) from openhands.sdk.logger import DEBUG, get_logger from openhands.sdk.tool.registry import seal_tool_catalog, unseal_tool_catalog from openhands.sdk.utils.redact import sanitize_dict @@ -802,6 +805,23 @@ async def _credential_binding_activation_required_handler( }, ) + @api.exception_handler(InstallationMetadataLockTimeoutError) + async def _installation_lock_timeout_handler( + _request: Request, _exc: InstallationMetadataLockTimeoutError + ) -> JSONResponse: + # Another plugin, skill or Canvas extension operation held the + # installation metadata lock for the whole timeout: busy, not broken. + # The message stays generic because the exception names a local path. + return JSONResponse( + status_code=503, + content={ + "detail": ( + "Another plugin, skill or extension operation is in " + "progress. Please retry." + ) + }, + ) + @api.exception_handler(RequestValidationError) async def _validation_exception_handler( request: Request, exc: RequestValidationError diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py b/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py index f39ef44dc81..0c7a68e4438 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions_router.py @@ -196,6 +196,7 @@ class UninstallCanvasExtensionResponse(BaseModel): 400: {"description": "Could not read canvas extension source"}, 409: {"description": "Canvas extension already installed (use force=true)"}, 422: {"description": "Invalid canvas extension (bad manifest, etc.)"}, + 503: {"description": "Another extension operation is in progress"}, }, ) async def install_canvas_extension_endpoint( @@ -262,7 +263,9 @@ async def install_canvas_extension_endpoint( @canvas_extensions_router.get( - "/installed", response_model=InstalledCanvasExtensionsListResponse + "/installed", + response_model=InstalledCanvasExtensionsListResponse, + responses={503: {"description": "Another extension operation is in progress"}}, ) def list_installed_canvas_extensions_endpoint() -> ( InstalledCanvasExtensionsListResponse @@ -302,7 +305,10 @@ def get_installed_canvas_extension_endpoint( @canvas_extensions_router.patch( "/installed/{extension_name}", response_model=UpdateCanvasExtensionStateResponse, - responses={404: {"description": "Canvas extension not installed"}}, + responses={ + 404: {"description": "Canvas extension not installed"}, + 503: {"description": "Another extension operation is in progress"}, + }, ) async def set_canvas_extension_enabled_endpoint( extension_name: CanvasExtensionNamePath, @@ -379,7 +385,10 @@ async def set_canvas_extension_enabled_endpoint( @canvas_extensions_router.delete( "/installed/{extension_name}", response_model=UninstallCanvasExtensionResponse, - responses={404: {"description": "Canvas extension not installed"}}, + responses={ + 404: {"description": "Canvas extension not installed"}, + 503: {"description": "Another extension operation is in progress"}, + }, ) async def uninstall_canvas_extension_endpoint( extension_name: CanvasExtensionNamePath, @@ -571,6 +580,7 @@ async def grant_canvas_extension_backend_secrets_endpoint( @canvas_extensions_router.delete( "/installed/{extension_name}/backend/secret-grant", response_model=BackendStatus, + responses={503: {"description": "Another extension operation is in progress"}}, ) async def revoke_canvas_extension_backend_secrets_endpoint( extension_name: CanvasExtensionNamePath, diff --git a/openhands-agent-server/openhands/agent_server/plugins_router.py b/openhands-agent-server/openhands/agent_server/plugins_router.py index 0d86c9ac7c3..784230b38e0 100644 --- a/openhands-agent-server/openhands/agent_server/plugins_router.py +++ b/openhands-agent-server/openhands/agent_server/plugins_router.py @@ -318,6 +318,7 @@ def fetch_plugin_endpoint(request: FetchPluginRequest) -> FetchPluginResponse: 400: {"description": "Failed to fetch plugin source"}, 409: {"description": "Plugin already installed (use force=true)"}, 422: {"description": "Invalid plugin (bad name, etc.)"}, + 503: {"description": "Another extension operation is in progress"}, }, ) def install_plugin_endpoint(request: InstallPluginRequest) -> InstalledPluginResponse: @@ -347,7 +348,11 @@ def install_plugin_endpoint(request: InstallPluginRequest) -> InstalledPluginRes ) -@plugins_router.get("/installed", response_model=InstalledPluginsListResponse) +@plugins_router.get( + "/installed", + response_model=InstalledPluginsListResponse, + responses={503: {"description": "Another extension operation is in progress"}}, +) def list_installed_plugins_endpoint() -> InstalledPluginsListResponse: """List all installed plugins (enabled and disabled).""" plugins = service_list_installed_plugins() @@ -377,7 +382,10 @@ def get_installed_plugin_endpoint( @plugins_router.patch( "/installed/{plugin_name}", response_model=UpdatePluginStateResponse, - responses={404: {"description": "Plugin not installed"}}, + responses={ + 404: {"description": "Plugin not installed"}, + 503: {"description": "Another extension operation is in progress"}, + }, ) def set_plugin_enabled_endpoint( plugin_name: PluginNamePath, request: UpdatePluginStateRequest @@ -395,7 +403,10 @@ def set_plugin_enabled_endpoint( @plugins_router.delete( "/installed/{plugin_name}", response_model=UninstallPluginResponse, - responses={404: {"description": "Plugin not installed"}}, + responses={ + 404: {"description": "Plugin not installed"}, + 503: {"description": "Another extension operation is in progress"}, + }, ) def uninstall_plugin_endpoint(plugin_name: PluginNamePath) -> UninstallPluginResponse: """Uninstall a plugin by name.""" @@ -413,6 +424,7 @@ def uninstall_plugin_endpoint(plugin_name: PluginNamePath) -> UninstallPluginRes responses={ 400: {"description": "Failed to fetch the plugin's recorded source"}, 404: {"description": "Plugin not installed"}, + 503: {"description": "Another extension operation is in progress"}, }, ) def refresh_plugin_endpoint(plugin_name: PluginNamePath) -> UpdatePluginResponse: @@ -449,7 +461,11 @@ def refresh_plugin_endpoint(plugin_name: PluginNamePath) -> UpdatePluginResponse ) -@plugins_router.get("/marketplace", response_model=MarketplaceCatalogResponse) +@plugins_router.get( + "/marketplace", + response_model=MarketplaceCatalogResponse, + responses={503: {"description": "Another extension operation is in progress"}}, +) def get_marketplace_catalog() -> MarketplaceCatalogResponse: """Get the plugins marketplace catalog with installation status. diff --git a/openhands-agent-server/openhands/agent_server/skills_router.py b/openhands-agent-server/openhands/agent_server/skills_router.py index b54abd9e3c8..b25a67cd089 100644 --- a/openhands-agent-server/openhands/agent_server/skills_router.py +++ b/openhands-agent-server/openhands/agent_server/skills_router.py @@ -378,6 +378,7 @@ def sync_skills() -> SyncResponse: 400: {"description": "Failed to fetch skill source"}, 409: {"description": "Skill already installed (use force=true)"}, 422: {"description": "Invalid skill (missing SKILL.md, etc.)"}, + 503: {"description": "Another extension operation is in progress"}, }, ) def install_skill_endpoint(request: InstallSkillRequest) -> InstalledSkillResponse: @@ -422,7 +423,11 @@ def install_skill_endpoint(request: InstallSkillRequest) -> InstalledSkillRespon ) -@skills_router.get("/installed", response_model=InstalledSkillsListResponse) +@skills_router.get( + "/installed", + response_model=InstalledSkillsListResponse, + responses={503: {"description": "Another extension operation is in progress"}}, +) def list_installed_skills_endpoint() -> InstalledSkillsListResponse: """List all installed skills. @@ -467,7 +472,10 @@ def get_installed_skill_endpoint(skill_name: SkillNamePath) -> InstalledSkillRes @skills_router.patch( "/installed/{skill_name}", response_model=UpdateSkillStateResponse, - responses={404: {"description": "Skill not installed"}}, + responses={ + 404: {"description": "Skill not installed"}, + 503: {"description": "Another extension operation is in progress"}, + }, ) def set_skill_enabled_endpoint( skill_name: SkillNamePath, request: UpdateSkillStateRequest @@ -500,7 +508,10 @@ def set_skill_enabled_endpoint( @skills_router.delete( "/installed/{skill_name}", response_model=UninstallSkillResponse, - responses={404: {"description": "Skill not installed"}}, + responses={ + 404: {"description": "Skill not installed"}, + 503: {"description": "Another extension operation is in progress"}, + }, ) def uninstall_skill_endpoint(skill_name: SkillNamePath) -> UninstallSkillResponse: """Uninstall a skill by name. @@ -533,6 +544,7 @@ def uninstall_skill_endpoint(skill_name: SkillNamePath) -> UninstallSkillRespons responses={ 400: {"description": "Failed to fetch the skill's recorded source"}, 404: {"description": "Skill not installed"}, + 503: {"description": "Another extension operation is in progress"}, }, ) def refresh_skill_endpoint(skill_name: SkillNamePath) -> UpdateSkillResponse: @@ -581,7 +593,11 @@ def refresh_skill_endpoint(skill_name: SkillNamePath) -> UpdateSkillResponse: ) -@skills_router.get("/marketplace", response_model=MarketplaceCatalogResponse) +@skills_router.get( + "/marketplace", + response_model=MarketplaceCatalogResponse, + responses={503: {"description": "Another extension operation is in progress"}}, +) def get_marketplace_catalog() -> MarketplaceCatalogResponse: """Get the marketplace catalog with installation status. diff --git a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py index b5626467313..c7d5f2a3839 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/__init__.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/__init__.py @@ -10,6 +10,7 @@ from openhands.sdk.extensions.installation.metadata import ( InstallationMetadata, InstallationMetadataLoadError, + InstallationMetadataLockTimeoutError, MetadataSession, ) @@ -21,6 +22,7 @@ "InstallationManager", "InstallationMetadata", "InstallationMetadataLoadError", + "InstallationMetadataLockTimeoutError", "InstallationOutcome", "MetadataSession", ] diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 59c03f7c2da..ecc97a666f0 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -37,6 +37,23 @@ def __init__(self, metadata_path: Path, cause: BaseException) -> None: self.metadata_path = metadata_path +class InstallationMetadataLockTimeoutError(Exception): + """The installation metadata lock was not acquired in time. + + Another operation (an install, update, uninstall or listing) held it for + the whole timeout. Retrying later can succeed. Deliberately not a + ``TimeoutError``: that is an ``OSError``, which callers already map to + unrelated errors (an invalid extension, for one). + """ + + def __init__(self, installed_dir: Path, timeout: float) -> None: + super().__init__( + f"Timed out after {timeout}s waiting for the installation " + f"metadata lock in {installed_dir}" + ) + self.installed_dir = installed_dir + + class MetadataSession: """Context manager that binds ``InstallationMetadata`` to its directory. @@ -165,6 +182,8 @@ def open( session will call ``sync()``. Raises: + InstallationMetadataLockTimeoutError: Another session held the + lock for the whole timeout. InstallationMetadataLoadError: The metadata file exists but cannot be read or parsed. The lock is released and the file is left unchanged. @@ -187,15 +206,14 @@ def _acquire_lock(installed_dir: Path) -> BaseFileLock: lock = FileLock(installed_dir / ".metadata.lock") try: lock.acquire(timeout=_LOCK_TIMEOUT_SECONDS) - except Timeout: + except Timeout as e: logger.error( f"Failed to acquire installation metadata lock in " f"{_LOCK_TIMEOUT_SECONDS}s: {installed_dir}" ) - raise TimeoutError( - "Installation metadata lock acquisition timed out after " - f"{_LOCK_TIMEOUT_SECONDS}s" - ) + raise InstallationMetadataLockTimeoutError( + installed_dir, _LOCK_TIMEOUT_SECONDS + ) from e return lock @classmethod diff --git a/tests/agent_server/test_installation_lock_timeout.py b/tests/agent_server/test_installation_lock_timeout.py new file mode 100644 index 00000000000..b83c4fa0c5f --- /dev/null +++ b/tests/agent_server/test_installation_lock_timeout.py @@ -0,0 +1,140 @@ +"""A plugin, skill or Canvas request that times out on the installation +metadata lock answers 503, not a 500 or a 422. + +Another holder keeps the real lock file locked while each endpoint runs, with +the lock timeout cut to 0.2 s. Goes through ``create_app`` because the mapping +is an app-level exception handler. +""" + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from filelock import FileLock + +from openhands.agent_server.api import create_app +from openhands.agent_server.config import Config +from openhands.sdk.extensions.installation import metadata as metadata_module + +from .canvas_extensions.conftest import write_extension + + +_BUSY_DETAIL = ( + "Another plugin, skill or extension operation is in progress. Please retry." +) + + +@pytest.fixture +def stores(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path]: + plugins = tmp_path / "plugins-installed" + skills = tmp_path / "skills-installed" + canvas = tmp_path / "canvas-installed" + for store in (plugins, skills, canvas): + store.mkdir() + monkeypatch.setattr( + "openhands.sdk.plugin.installed.DEFAULT_INSTALLED_PLUGINS_DIR", plugins + ) + monkeypatch.setattr( + "openhands.sdk.skills.installed.DEFAULT_INSTALLED_SKILLS_DIR", skills + ) + monkeypatch.setattr( + "openhands.agent_server.canvas_extensions.installed." + "get_installed_canvas_extensions_dir", + lambda: canvas, + ) + # The marketplace routes fetch their catalog with git before listing. + monkeypatch.setattr( + "openhands.agent_server.plugins_service._fetch_plugin_catalog_entries", + lambda *args, **kwargs: [], + ) + monkeypatch.setattr( + "openhands.agent_server.skills_service._fetch_catalog_entries", + lambda *args, **kwargs: [], + ) + return {"plugins": plugins, "skills": skills, "canvas": canvas} + + +@pytest.fixture +def client(tmp_path: Path, stores: dict[str, Path]) -> TestClient: + client = TestClient( + create_app(Config(session_api_keys=[])), raise_server_exceptions=False + ) + plugin = tmp_path / "src" / "demo-plugin" + (plugin / ".plugin").mkdir(parents=True) + (plugin / ".plugin" / "plugin.json").write_text( + json.dumps({"name": "demo-plugin", "version": "1.0.0", "description": "x"}) + ) + skill = tmp_path / "src" / "demo-skill" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text( + "---\nname: demo-skill\ndescription: A test skill\n---\nBody\n" + ) + canvas = write_extension(tmp_path / "src" / "demo-canvas", name="demo-canvas") + for route, source in ( + ("/api/plugins/install", plugin), + ("/api/skills/install", skill), + ("/api/canvas-extensions/install", canvas), + ): + response = client.post(route, json={"source": str(source)}) + assert response.status_code == 200, response.text + return client + + +@pytest.mark.parametrize( + ("store", "method", "path", "json_body"), + [ + ("plugins", "POST", "/api/plugins/install", "source"), + ("plugins", "GET", "/api/plugins/installed", None), + ("plugins", "PATCH", "/api/plugins/installed/demo-plugin", {"enabled": False}), + ("plugins", "DELETE", "/api/plugins/installed/demo-plugin", None), + ("plugins", "POST", "/api/plugins/installed/demo-plugin/refresh", None), + ("plugins", "GET", "/api/plugins/marketplace", None), + ("skills", "POST", "/api/skills/install", "source"), + ("skills", "GET", "/api/skills/installed", None), + ("skills", "PATCH", "/api/skills/installed/demo-skill", {"enabled": False}), + ("skills", "DELETE", "/api/skills/installed/demo-skill", None), + ("skills", "POST", "/api/skills/installed/demo-skill/refresh", None), + ("skills", "GET", "/api/skills/marketplace", None), + ("canvas", "POST", "/api/canvas-extensions/install", "source"), + ("canvas", "GET", "/api/canvas-extensions/installed", None), + ( + "canvas", + "PATCH", + "/api/canvas-extensions/installed/demo-canvas", + {"enabled": True}, + ), + ("canvas", "DELETE", "/api/canvas-extensions/installed/demo-canvas", None), + ( + "canvas", + "DELETE", + "/api/canvas-extensions/installed/demo-canvas/backend/secret-grant", + None, + ), + ], +) +def test_lock_timeout_returns_503( + client: TestClient, + stores: dict[str, Path], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + store: str, + method: str, + path: str, + json_body: dict | str | None, +): + if json_body == "source": + # A force reinstall from a fresh source, so the request gets past the + # "already installed" check and fetching, up to the lock. + name = {"plugins": "demo-plugin", "skills": "demo-skill"}.get( + store, "demo-canvas" + ) + json_body = {"source": str(tmp_path / "src" / name), "force": True} + monkeypatch.setattr(metadata_module, "_LOCK_TIMEOUT_SECONDS", 0.2) + + held = FileLock(stores[store] / ".metadata.lock") + with held: + response = client.request(method, path, json=json_body) + + assert response.status_code == 503, response.text + assert response.json() == {"detail": _BUSY_DETAIL} diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py index cbbacb0fb3c..b54882c1bf4 100644 --- a/tests/sdk/extensions/installation/test_installation_concurrency.py +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -15,6 +15,7 @@ InstallationInterface, InstallationManager, InstallationMetadata, + InstallationMetadataLockTimeoutError, MetadataSession, manager as manager_module, metadata as metadata_module, @@ -227,7 +228,11 @@ def test_session_lock_times_out_instead_of_blocking_forever( ) assert len(errors) == 1 - assert isinstance(errors[0], TimeoutError) + assert isinstance(errors[0], InstallationMetadataLockTimeoutError) + # Not an OSError: the Canvas install endpoint maps OSError to 422 + # "Invalid canvas extension". + assert not isinstance(errors[0], OSError) + assert errors[0].installed_dir == installation_dir def _raise_runtime_error(*args, **kwargs): From 84c832e3e7f6ccb9e19aea731e2bd240129978b5 Mon Sep 17 00:00:00 2001 From: adithya Date: Wed, 11 Nov 2026 20:03:22 +0530 Subject: [PATCH 11/15] fix(sdk): keep the installation metadata file's mode on save Saves forced 0o600, narrowing a metadata file other local users could read. An existing file now keeps its mode, and a new file gets the umask's mode as a plain write gives it, as before saves became atomic. atomic_write_text gains mode=None for that; its 0o600 default and its other callers are unchanged. metadata_session becomes a method: it takes the metadata lock, which an attribute access should not do. --- .../canvas_extensions/installed.py | 2 +- .../sdk/extensions/installation/manager.py | 14 +++-- .../sdk/extensions/installation/metadata.py | 13 +++- openhands-sdk/openhands/sdk/utils/files.py | 60 +++++++++++++++---- .../test_installation_metadata.py | 40 +++++++++++-- tests/sdk/utils/test_files.py | 33 +++++++++- 6 files changed, 135 insertions(+), 27 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py index c1bd68338af..1b22f8ab1a8 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py @@ -454,7 +454,7 @@ def apply_canvas_extension_update( # The swap and the metadata write share one session, so a concurrent # list can't see the install path missing between the two renames # and prune the entry. - with manager.metadata_session as session: + with manager.metadata_session() as session: current_info = session.extensions.get(name) if current_info is None or not install_path.is_dir(): return None diff --git a/openhands-sdk/openhands/sdk/extensions/installation/manager.py b/openhands-sdk/openhands/sdk/extensions/installation/manager.py index 9c486f1c5ef..38b34f99a74 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/manager.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/manager.py @@ -81,9 +81,11 @@ class InstallationManager[T: ExtensionProtocol]: def __post_init__(self) -> None: self.installation_dir = self.installation_dir.resolve() - @property def metadata_session(self) -> MetadataSession: - """Open a metadata session bound to this manager's dir and interface.""" + """Open a metadata session bound to this manager's dir and interface. + + Takes the metadata lock when called; use it in a ``with`` statement. + """ return InstallationMetadata.open( self.installation_dir, interface=self.installation_interface ) @@ -199,7 +201,7 @@ def _install( staged_path = staging_dir / "new" shutil.copytree(fetched_path, staged_path) - with self.metadata_session as session: + with self.metadata_session() as session: if install_path.exists() and not force: raise FileExistsError( f"Extension '{extension.name}' is already installed" @@ -273,7 +275,7 @@ def uninstall(self, name: str) -> bool: """ validate_extension_name(name) - with self.metadata_session as session: + with self.metadata_session() as session: if name not in session.extensions: logger.warning(f"Extension '{name}' is not installed") return False @@ -311,7 +313,7 @@ def _set_enabled( ) return False - with self.metadata_session as session: + with self.metadata_session() as session: session.sync() info = session.extensions.get(name) @@ -357,7 +359,7 @@ def list_installed(self) -> list[InstallationInfo]: if not self.installation_dir.exists(): return [] - with self.metadata_session as session: + with self.metadata_session() as session: return session.sync() def load_installed(self) -> list[T]: diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index ecc97a666f0..08f05c830ad 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -1,5 +1,6 @@ from __future__ import annotations +import stat from pathlib import Path from types import TracebackType from typing import Any, ClassVar, Final @@ -261,10 +262,18 @@ def _load_for_session(cls, installed_dir: Path) -> InstallationMetadata: raise InstallationMetadataLoadError(metadata_path, e) from e def save_to_dir(self, installed_dir: Path) -> None: - """Save metadata to the installed extensions directory.""" + """Save metadata to the installed extensions directory. + + An existing file keeps its permission bits. A new file gets them from + the umask, as it did before saves became atomic. + """ metadata_path = self.get_metadata_path(installed_dir) metadata_path.parent.mkdir(parents=True, exist_ok=True) - atomic_write_text(metadata_path, self.model_dump_json(indent=2), mode=0o600) + try: + mode: int | None = stat.S_IMODE(metadata_path.stat().st_mode) + except FileNotFoundError: + mode = None + atomic_write_text(metadata_path, self.model_dump_json(indent=2), mode=mode) def validate_tracked(self, installed_dir: Path) -> list[InstallationInfo]: """Validate tracked extensions exist on disk. diff --git a/openhands-sdk/openhands/sdk/utils/files.py b/openhands-sdk/openhands/sdk/utils/files.py index 4e3c0d1eeec..5cf282c3e32 100644 --- a/openhands-sdk/openhands/sdk/utils/files.py +++ b/openhands-sdk/openhands/sdk/utils/files.py @@ -1,4 +1,6 @@ import os +import secrets +import sys import tempfile import time from collections.abc import Callable @@ -52,18 +54,56 @@ def read_text_retrying(path: Path) -> str: return _retry_on_permission_error(lambda: path.read_text(encoding="utf-8")) -def atomic_write_text(path: Path, value: str, mode: int = 0o600) -> None: - """Atomically write text with owner-only permissions.""" - fd, temporary_name = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=_TEMP_SUFFIX, dir=path.parent - ) +# ``tempfile.mkstemp``'s flags, for a temp file it can't create: one whose mode +# the umask decides. Windows has no ``O_NOFOLLOW`` and needs ``O_BINARY`` to +# keep the C runtime from translating newlines a second time. +_TEMP_OPEN_FLAGS: Final[int] = ( + os.O_RDWR + | os.O_CREAT + | os.O_EXCL + | (os.O_BINARY if sys.platform == "win32" else os.O_NOFOLLOW) +) + + +def _create_temp_file_with_umask_mode(path: Path) -> tuple[int, str]: + """Create a temp file next to ``path`` the way ``open()`` creates a file. + + Like ``tempfile.mkstemp``, but the file starts as ``0o666`` and the umask + removes bits from it, where ``mkstemp`` always creates ``0o600``. The umask + itself can only be read by setting it, which would change it for every + thread in the process for that moment. + """ + for _ in range(100): + name = os.path.join( + path.parent, f".{path.name}.{secrets.token_hex(8)}{_TEMP_SUFFIX}" + ) + try: + return os.open(name, _TEMP_OPEN_FLAGS, 0o666), name + except FileExistsError: + continue + raise FileExistsError(f"No unused temporary file name next to {path}") + + +def atomic_write_text(path: Path, value: str, mode: int | None = 0o600) -> None: + """Atomically write text to ``path`` by replacing it. + + ``mode`` is the permission bits ``path`` ends up with: owner-only by + default. ``None`` leaves them to the umask, as for a file ``open()`` creates. + """ + if mode is None: + fd, temporary_name = _create_temp_file_with_umask_mode(path) + else: + fd, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=_TEMP_SUFFIX, dir=path.parent + ) temporary_path = Path(temporary_name) try: - fchmod = getattr(os, "fchmod", None) - if fchmod is None: - os.chmod(temporary_path, mode) - else: - fchmod(fd, mode) + if mode is not None: + fchmod = getattr(os, "fchmod", None) + if fchmod is None: + os.chmod(temporary_path, mode) + else: + fchmod(fd, mode) file = os.fdopen(fd, "w", encoding="utf-8") fd = -1 with file: diff --git a/tests/sdk/extensions/installation/test_installation_metadata.py b/tests/sdk/extensions/installation/test_installation_metadata.py index 3bf2e5c9728..b45e15989e0 100644 --- a/tests/sdk/extensions/installation/test_installation_metadata.py +++ b/tests/sdk/extensions/installation/test_installation_metadata.py @@ -442,13 +442,41 @@ def test_discover_untracked_skips_mismatched_manifest_name(tmp_path: Path): @pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") -def test_save_to_dir_writes_owner_only_file(tmp_path: Path): - """Sources can carry URL credentials, so the metadata file must not be - readable by group or other users, even if an existing file was wider.""" +@pytest.mark.parametrize("existing_mode", [0o600, 0o644], ids=oct) +def test_save_to_dir_keeps_existing_file_mode(tmp_path: Path, existing_mode: int): + """A save keeps the file's mode whatever the umask: an owner-only file stays + owner-only (sources can carry URL credentials), and a file other local users + read is not narrowed under them.""" metadata_path = InstallationMetadata.get_metadata_path(tmp_path) metadata_path.write_text("{}") - metadata_path.chmod(0o644) + metadata_path.chmod(existing_mode) + previous_umask = os.umask(0o027) + try: + InstallationMetadata().save_to_dir(tmp_path) + finally: + os.umask(previous_umask) + + assert stat.S_IMODE(metadata_path.stat().st_mode) == existing_mode - InstallationMetadata().save_to_dir(tmp_path) - assert stat.S_IMODE(metadata_path.stat().st_mode) == 0o600 +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.parametrize( + ("umask", "expected"), + [ + pytest.param(0o022, 0o644, id="umask-022"), + pytest.param(0o077, 0o600, id="umask-077"), + ], +) +def test_save_to_dir_creates_new_file_with_umask_mode( + tmp_path: Path, umask: int, expected: int +): + """A new file gets the umask's mode, as a plain write would give it: a + strict umask keeps sources, which can carry URL credentials, owner-only.""" + metadata_path = InstallationMetadata.get_metadata_path(tmp_path) + previous_umask = os.umask(umask) + try: + InstallationMetadata().save_to_dir(tmp_path) + finally: + os.umask(previous_umask) + + assert stat.S_IMODE(metadata_path.stat().st_mode) == expected diff --git a/tests/sdk/utils/test_files.py b/tests/sdk/utils/test_files.py index fca2b33f78b..45c90503747 100644 --- a/tests/sdk/utils/test_files.py +++ b/tests/sdk/utils/test_files.py @@ -1,4 +1,6 @@ import os +import stat +import sys from pathlib import Path import pytest @@ -11,8 +13,9 @@ ) +@pytest.mark.parametrize("mode", [0o600, None], ids=["mkstemp", "umask"]) def test_atomic_write_text_temp_file_is_marked_as_a_temp_file( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: int | None ) -> None: temp_paths: list[Path] = [] real_replace = os.replace @@ -25,7 +28,7 @@ def recording_replace(src, dst): names = ("base_state.json", "event-00000-abc.json", ".eventlog-len-3.marker") for name in names: - atomic_write_text(tmp_path / name, "{}") + atomic_write_text(tmp_path / name, "{}", mode=mode) assert [path.parent for path in temp_paths] == [tmp_path] * len(names) for name, temp_path in zip(names, temp_paths, strict=True): @@ -35,6 +38,32 @@ def recording_replace(src, dst): assert sorted(path.name for path in tmp_path.iterdir()) == sorted(names) +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +@pytest.mark.parametrize( + ("umask", "mode", "expected"), + [ + pytest.param(0o022, 0o600, 0o600, id="default-owner-only"), + pytest.param(0o022, 0o640, 0o640, id="explicit"), + pytest.param(0o022, None, 0o644, id="umask-022"), + pytest.param(0o077, None, 0o600, id="umask-077"), + ], +) +def test_atomic_write_text_mode( + tmp_path: Path, umask: int, mode: int | None, expected: int +) -> None: + path = tmp_path / "data.json" + path.write_text("old") + path.chmod(0o604) + previous_umask = os.umask(umask) + try: + atomic_write_text(path, "new", mode=mode) + finally: + os.umask(previous_umask) + + assert stat.S_IMODE(path.stat().st_mode) == expected + assert path.read_text() == "new" + + @pytest.mark.parametrize( "name", [ From 82c532068651c703bcfd947a7922286811966459 Mon Sep 17 00:00:00 2001 From: adithya Date: Wed, 11 Nov 2026 21:04:02 +0530 Subject: [PATCH 12/15] fix(sdk): keep the user's ownership when root saves installation metadata A root process sharing an installation directory with its user (an agent-server container on a bind mount) left a root-owned .metadata.lock the user could no longer open, and replaced the user's metadata file with a root-owned one. When running as root, give the lock file to the directory's owner and keep the metadata file's previous owner, as the in-place write did before saves became atomic. Best effort and POSIX only. The lock file is changed through a descriptor opened without following symlinks, and only when it has a single link. --- .../sdk/extensions/installation/metadata.py | 61 +++++++++++++++++-- openhands-sdk/openhands/sdk/utils/files.py | 20 +++++- .../test_installation_metadata.py | 51 ++++++++++++++++ tests/sdk/utils/test_files.py | 18 ++++++ 4 files changed, 143 insertions(+), 7 deletions(-) diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index 08f05c830ad..c1621d09eda 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -1,6 +1,8 @@ from __future__ import annotations +import os import stat +import sys from pathlib import Path from types import TracebackType from typing import Any, ClassVar, Final @@ -22,6 +24,40 @@ _LOCK_TIMEOUT_SECONDS: Final[float] = 30.0 +def _running_as_root() -> bool: + return sys.platform != "win32" and os.geteuid() == 0 + + +def _give_lock_file_to_directory_owner(lock_path: Path) -> None: + """Make the installation directory's owner own the lock file root holds. + + A root process (an agent-server container, say) sharing the directory with + its user would otherwise leave a root-owned lock file that user can never + open again. The file is changed through a descriptor opened without + following symlinks, and only if it is a regular file with a single link, + so the directory's owner can't redirect the change to another file. Best + effort: a failure is logged and the session goes on. + """ + if sys.platform == "win32": + return + try: + directory = lock_path.parent.stat() + fd = os.open(lock_path, os.O_RDONLY | os.O_NOFOLLOW) + try: + current = os.fstat(fd) + if ( + stat.S_ISREG(current.st_mode) + and current.st_nlink == 1 + and (current.st_uid, current.st_gid) + != (directory.st_uid, directory.st_gid) + ): + os.fchown(fd, directory.st_uid, directory.st_gid) + finally: + os.close(fd) + except OSError as e: + logger.warning(f"Could not give {lock_path} to its directory's owner: {e}") + + class InstallationMetadataLoadError(Exception): """An existing metadata file could not be read or parsed. @@ -204,7 +240,8 @@ def open( @staticmethod def _acquire_lock(installed_dir: Path) -> BaseFileLock: - lock = FileLock(installed_dir / ".metadata.lock") + lock_path = installed_dir / ".metadata.lock" + lock = FileLock(lock_path) try: lock.acquire(timeout=_LOCK_TIMEOUT_SECONDS) except Timeout as e: @@ -215,6 +252,8 @@ def _acquire_lock(installed_dir: Path) -> BaseFileLock: raise InstallationMetadataLockTimeoutError( installed_dir, _LOCK_TIMEOUT_SECONDS ) from e + if _running_as_root(): + _give_lock_file_to_directory_owner(lock_path) return lock @classmethod @@ -264,16 +303,26 @@ def _load_for_session(cls, installed_dir: Path) -> InstallationMetadata: def save_to_dir(self, installed_dir: Path) -> None: """Save metadata to the installed extensions directory. - An existing file keeps its permission bits. A new file gets them from - the umask, as it did before saves became atomic. + An existing file keeps its permission bits, and, when root saves it, its + owner. A new file gets its mode from the umask. Both as before saves + became atomic, when the file was rewritten in place. """ metadata_path = self.get_metadata_path(installed_dir) metadata_path.parent.mkdir(parents=True, exist_ok=True) + mode: int | None = None + owner: tuple[int, int] | None = None try: - mode: int | None = stat.S_IMODE(metadata_path.stat().st_mode) + existing = metadata_path.lstat() except FileNotFoundError: - mode = None - atomic_write_text(metadata_path, self.model_dump_json(indent=2), mode=mode) + pass + else: + if stat.S_ISREG(existing.st_mode): + mode = stat.S_IMODE(existing.st_mode) + if _running_as_root(): + owner = (existing.st_uid, existing.st_gid) + atomic_write_text( + metadata_path, self.model_dump_json(indent=2), mode=mode, owner=owner + ) def validate_tracked(self, installed_dir: Path) -> list[InstallationInfo]: """Validate tracked extensions exist on disk. diff --git a/openhands-sdk/openhands/sdk/utils/files.py b/openhands-sdk/openhands/sdk/utils/files.py index 5cf282c3e32..4450b083de7 100644 --- a/openhands-sdk/openhands/sdk/utils/files.py +++ b/openhands-sdk/openhands/sdk/utils/files.py @@ -7,6 +7,10 @@ from pathlib import Path from typing import Final +from openhands.sdk.logger import get_logger + + +logger = get_logger(__name__) # Suffix that marks a file as still being written. ``atomic_write_text`` writes # to ``...tmp`` next to ``path`` and then renames it to ````. @@ -84,11 +88,20 @@ def _create_temp_file_with_umask_mode(path: Path) -> tuple[int, str]: raise FileExistsError(f"No unused temporary file name next to {path}") -def atomic_write_text(path: Path, value: str, mode: int | None = 0o600) -> None: +def atomic_write_text( + path: Path, + value: str, + mode: int | None = 0o600, + owner: tuple[int, int] | None = None, +) -> None: """Atomically write text to ``path`` by replacing it. ``mode`` is the permission bits ``path`` ends up with: owner-only by default. ``None`` leaves them to the umask, as for a file ``open()`` creates. + + ``owner`` is a ``(uid, gid)`` for ``path`` to end up with, on POSIX. It is + best effort: if it can't be applied, ``path`` keeps the writer's owner and + a warning is logged. """ if mode is None: fd, temporary_name = _create_temp_file_with_umask_mode(path) @@ -104,6 +117,11 @@ def atomic_write_text(path: Path, value: str, mode: int | None = 0o600) -> None: os.chmod(temporary_path, mode) else: fchmod(fd, mode) + if owner is not None and sys.platform != "win32": + try: + os.fchown(fd, *owner) + except OSError as e: + logger.warning(f"Could not keep the owner of {path}: {e}") file = os.fdopen(fd, "w", encoding="utf-8") fd = -1 with file: diff --git a/tests/sdk/extensions/installation/test_installation_metadata.py b/tests/sdk/extensions/installation/test_installation_metadata.py index b45e15989e0..da0687f473d 100644 --- a/tests/sdk/extensions/installation/test_installation_metadata.py +++ b/tests/sdk/extensions/installation/test_installation_metadata.py @@ -1,5 +1,6 @@ import logging import os +import shutil import stat import sys from pathlib import Path @@ -480,3 +481,53 @@ def test_save_to_dir_creates_new_file_with_umask_mode( os.umask(previous_umask) assert stat.S_IMODE(metadata_path.stat().st_mode) == expected + + +# A uid and gid that are not root's: the user a root process (an agent-server +# container) shares an installation directory with. +_USER = 65534 + +_requires_root = pytest.mark.skipif( + getattr(os, "geteuid", lambda: -1)() != 0, reason="changing owners needs root" +) + + +@_requires_root +def test_root_session_keeps_the_directory_users_ownership(tmp_path: Path): + """When root saves in a user's installation directory, that user can still + read the metadata file and take the lock afterwards.""" + installation_dir = tmp_path / "installed" + _write_mock_extension(installation_dir / "mock-extension") + shutil.chown(installation_dir, _USER, _USER) + metadata_path = InstallationMetadata.get_metadata_path(installation_dir) + metadata_path.write_text('{"extensions": {}}') + metadata_path.chmod(0o600) + shutil.chown(metadata_path, _USER, _USER) + + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ) as session: + session.sync() + + for path in (metadata_path, installation_dir / ".metadata.lock"): + assert (path.stat().st_uid, path.stat().st_gid) == (_USER, _USER), path + assert stat.S_IMODE(metadata_path.stat().st_mode) == 0o600 + + +@_requires_root +def test_root_session_does_not_give_away_a_hard_linked_lock_file(tmp_path: Path): + """A lock file that is another file's second name is left alone: the + directory's owner must not be able to get that file through root.""" + installation_dir = tmp_path / "installed" + installation_dir.mkdir() + shutil.chown(installation_dir, _USER, _USER) + other_file = tmp_path / "root-owned" + other_file.write_text("x") + os.link(other_file, installation_dir / ".metadata.lock") + + with InstallationMetadata.open( + installation_dir, interface=MockExtensionInstallationInterface() + ): + pass + + assert other_file.stat().st_uid == 0 diff --git a/tests/sdk/utils/test_files.py b/tests/sdk/utils/test_files.py index 45c90503747..51d584fbaeb 100644 --- a/tests/sdk/utils/test_files.py +++ b/tests/sdk/utils/test_files.py @@ -1,3 +1,4 @@ +import logging import os import stat import sys @@ -64,6 +65,23 @@ def test_atomic_write_text_mode( assert path.read_text() == "new" +@pytest.mark.skipif( + sys.platform == "win32" or getattr(os, "geteuid", lambda: -1)() == 0, + reason="needs a user who may not give files to root", +) +def test_atomic_write_text_owner_is_best_effort( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + path = tmp_path / "data.json" + + with caplog.at_level(logging.WARNING): + atomic_write_text(path, "new", owner=(0, 0)) + + assert path.read_text() == "new" + assert path.stat().st_uid == tmp_path.stat().st_uid != 0 + assert any("Could not keep the owner" in r.message for r in caplog.records) + + @pytest.mark.parametrize( "name", [ From 15d52e6a1ecbbba4550cfc710d6e576f25018257 Mon Sep 17 00:00:00 2001 From: adithya Date: Wed, 11 Nov 2026 23:17:02 +0530 Subject: [PATCH 13/15] test(agent-server): stub the marketplace clone in the lock-timeout tests --- .../test_installation_lock_timeout.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/agent_server/test_installation_lock_timeout.py b/tests/agent_server/test_installation_lock_timeout.py index b83c4fa0c5f..7aa435ada47 100644 --- a/tests/agent_server/test_installation_lock_timeout.py +++ b/tests/agent_server/test_installation_lock_timeout.py @@ -43,15 +43,14 @@ def stores(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path]: "get_installed_canvas_extensions_dir", lambda: canvas, ) - # The marketplace routes fetch their catalog with git before listing. - monkeypatch.setattr( - "openhands.agent_server.plugins_service._fetch_plugin_catalog_entries", - lambda *args, **kwargs: [], - ) - monkeypatch.setattr( - "openhands.agent_server.skills_service._fetch_catalog_entries", - lambda *args, **kwargs: [], - ) + # The marketplace routes clone their catalog with git before listing. + empty_repo = tmp_path / "empty-marketplace-repo" + empty_repo.mkdir() + for service in ("plugins_service", "skills_service"): + monkeypatch.setattr( + f"openhands.agent_server.{service}.update_skills_repository", + lambda *args, **kwargs: empty_repo, + ) return {"plugins": plugins, "skills": skills, "canvas": canvas} From ab876c017dadab568c3790967cdbaa20f2ca7635 Mon Sep 17 00:00:00 2001 From: adithya Date: Thu, 12 Nov 2026 07:56:59 +0530 Subject: [PATCH 14/15] fix(agent-server): decide which listed Canvas extensions are new under the listing's lock A list racing a force reinstall could take the installed extension for a new one and disable it. Also drops an unused test helper. --- .../canvas_extensions/installed.py | 25 +++----- .../test_canvas_extensions_installed.py | 59 +++++++++++++++++++ .../test_installation_concurrency.py | 7 --- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py index 1b22f8ab1a8..fe6c12b18a0 100644 --- a/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py +++ b/openhands-agent-server/openhands/agent_server/canvas_extensions/installed.py @@ -35,7 +35,6 @@ InstallationInfo, InstallationInterface, InstallationManager, - InstallationMetadata, ) from openhands.sdk.extensions.installation.manager import DEFAULT_CACHE_DIR from openhands.sdk.logger import get_logger @@ -82,19 +81,6 @@ def _manager(installed_dir: Path) -> InstallationManager[CanvasExtensionManifest ) -def _tracked_names(installed_dir: Path) -> set[str]: - """Names with a real, valid tracked entry, not just a metadata key. - - A stale record with no matching directory must not count as "already - installed" -- otherwise a real install of that name would wrongly - inherit its state as if this were a force-reinstall. - """ - if not installed_dir.exists(): - return set() - metadata = InstallationMetadata.load_from_dir(installed_dir) - return {info.name for info in metadata.validate_tracked(installed_dir)} - - def _force_disable_new( manager: InstallationManager[CanvasExtensionManifest], info: InstallationInfo, @@ -181,9 +167,16 @@ def list_installed_canvas_extensions( discovered this way also land disabled. """ installed_dir = _resolve_installed_dir(installed_dir) + if not installed_dir.exists(): + return [] manager = _manager(installed_dir) - pre_existing = _tracked_names(installed_dir) - infos = manager.list_installed() + # Decided under the lock the listing takes: a reinstall swapping the + # directory in can't make an installed extension look new meanwhile. + with manager.metadata_session() as session: + pre_existing = { + info.name for info in session.metadata.validate_tracked(installed_dir) + } + infos = session.sync() return [_force_disable_new(manager, info, pre_existing) for info in infos] diff --git a/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py b/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py index 65089d08da7..12ee1818a91 100644 --- a/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py +++ b/tests/agent_server/canvas_extensions/test_canvas_extensions_installed.py @@ -8,6 +8,7 @@ import inspect import json +import os import shutil import threading from pathlib import Path @@ -17,6 +18,7 @@ from pydantic import ValidationError from openhands.agent_server.canvas_extensions.installed import ( + InstalledCanvasExtensionInfo, disable_canvas_extension, enable_canvas_extension, get_installed_canvas_extension, @@ -475,3 +477,60 @@ def copytree(src, dst, *args, **kwargs): assert info.enabled is False on_disk = InstallationMetadata.load_from_dir(installed_dir) assert on_disk.extensions["my-extension"].enabled is False + + +def test_list_during_a_reinstall_keeps_an_enabled_extension_enabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """A list landing while a force reinstall swaps the directory in must not + take the installed extension for a new one and disable it. + """ + installed_dir = tmp_path / "installed" + source = _write_extension( + tmp_path / "src" / "demo-extension", name="demo-extension" + ) + install_canvas_extension(source=str(source), installed_dir=installed_dir) + assert enable_canvas_extension("demo-extension", installed_dir=installed_dir) + + list_waiting = threading.Event() + acquire_lock = InstallationMetadata._acquire_lock + + def signalling_acquire_lock(directory: Path): + if threading.current_thread().name == "list": + list_waiting.set() + return acquire_lock(directory) + + monkeypatch.setattr( + InstallationMetadata, "_acquire_lock", staticmethod(signalling_acquire_lock) + ) + real_replace = os.replace + lists: list[threading.Thread] = [] + listed: list[list[InstalledCanvasExtensionInfo]] = [] + + def replace(src, dst) -> None: + real_replace(src, dst) + if Path(src) == installed_dir / "demo-extension" and not lists: + thread = threading.Thread( + target=lambda: listed.append( + list_installed_canvas_extensions(installed_dir=installed_dir) + ), + name="list", + daemon=True, + ) + lists.append(thread) + thread.start() + assert list_waiting.wait(5) + + monkeypatch.setattr(installation_manager.os, "replace", replace) + + install_canvas_extension( + source=str(source), installed_dir=installed_dir, force=True + ) + + assert len(lists) == 1, "the reinstall did not move the installed directory aside" + lists[0].join(10) + assert not lists[0].is_alive(), "the concurrent list did not finish" + assert [(i.name, i.enabled) for i in listed[0]] == [("demo-extension", True)] + info = get_installed_canvas_extension("demo-extension", installed_dir=installed_dir) + assert info is not None + assert info.enabled is True diff --git a/tests/sdk/extensions/installation/test_installation_concurrency.py b/tests/sdk/extensions/installation/test_installation_concurrency.py index b54882c1bf4..0c59e484523 100644 --- a/tests/sdk/extensions/installation/test_installation_concurrency.py +++ b/tests/sdk/extensions/installation/test_installation_concurrency.py @@ -90,13 +90,6 @@ def run() -> None: return errors -def _source_of(installation_dir: Path, name: str) -> str: - data = json.loads( - InstallationMetadata.get_metadata_path(installation_dir).read_text() - ) - return data["extensions"][name]["source"] - - @pytest.mark.parametrize("operation", ["install", "disable"]) def test_list_cannot_load_metadata_while_another_session_is_open( manager: InstallationManager[MockExtension], From 59e9ad15525ef4a20d17e107bef92d4025ebe158 Mon Sep 17 00:00:00 2001 From: adithya Date: Thu, 12 Nov 2026 08:40:43 +0530 Subject: [PATCH 15/15] docs(sdk): describe the metadata save rule without its history --- .../openhands/sdk/extensions/installation/metadata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py index c1621d09eda..f14e3c22305 100644 --- a/openhands-sdk/openhands/sdk/extensions/installation/metadata.py +++ b/openhands-sdk/openhands/sdk/extensions/installation/metadata.py @@ -303,9 +303,8 @@ def _load_for_session(cls, installed_dir: Path) -> InstallationMetadata: def save_to_dir(self, installed_dir: Path) -> None: """Save metadata to the installed extensions directory. - An existing file keeps its permission bits, and, when root saves it, its - owner. A new file gets its mode from the umask. Both as before saves - became atomic, when the file was rewritten in place. + An existing regular file keeps its permission bits, and, when root saves + it, its owner. A new file gets its mode from the umask. """ metadata_path = self.get_metadata_path(installed_dir) metadata_path.parent.mkdir(parents=True, exist_ok=True)