From e9268476f55bb90e64903fd896d00915cb835575 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:28:07 +0530 Subject: [PATCH 01/11] fix: validate Ollama blobs and surface incomplete scans - report enumeration errors instead of treating unreadable manifests as removed models - reject blobs whose size differs from the manifest and handle read errors - cover malformed digests, truncated blobs, and incomplete source scans Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. --- src/studiolink/ollama_adapter.py | 33 +++++++++++++++++----------- tests/conftest.py | 2 +- tests/test_ollama_adapter.py | 37 +++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/src/studiolink/ollama_adapter.py b/src/studiolink/ollama_adapter.py index 906d891..e5deaf7 100644 --- a/src/studiolink/ollama_adapter.py +++ b/src/studiolink/ollama_adapter.py @@ -2,6 +2,7 @@ import json import logging +import os import re from pathlib import Path @@ -36,9 +37,15 @@ def scan_report(self) -> OllamaScanReport: models: list[OllamaModel] = [] errors: list[str] = [] + + def scan_error(exc: OSError) -> None: + raise exc + try: manifest_files = sorted( - path for path in manifests_dir.rglob("*") if path.is_file() + Path(directory) / name + for directory, _, names in os.walk(manifests_dir, onerror=scan_error) + for name in names ) except OSError as exc: message = f"Could not enumerate Ollama manifests: {exc}" @@ -145,18 +152,20 @@ def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: ":", "-" ) logger.debug("Looking for blob: %s", blob_path) - if not blob_path.exists(): - logger.debug("Blob not found: %s", blob_path) - issues.append("model blob is missing from the Ollama blob store") - else: + try: blob_size = blob_path.stat().st_size - logger.debug("Found blob: %s (size=%d bytes)", blob_path, blob_size) - if self._has_gguf_header(blob_path): - logger.debug("Blob has valid GGUF header: %s", blob_path) - gguf_valid = True - else: - logger.debug("Blob missing GGUF magic bytes: %s", blob_path) + gguf_valid = self._has_gguf_header(blob_path) + if not gguf_valid: issues.append("blob does not start with GGUF magic bytes") + if declared_size is not None and blob_size != declared_size: + issues.append( + f"blob size mismatch: expected {declared_size}, found {blob_size}" + ) + gguf_valid = False + except FileNotFoundError: + issues.append("model blob is missing from the Ollama blob store") + except OSError as exc: + issues.append(f"could not read model blob: {exc}") canonical_name = self._canonical_name(registry, namespace, repository, tag) fully_qualified_name = f"{registry}/{namespace}/{repository}:{tag}" @@ -182,7 +191,7 @@ def _as_int(value: object) -> int | None: return None try: return int(value) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): return None @staticmethod diff --git a/tests/conftest.py b/tests/conftest.py index 11c5417..71194e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,7 +86,7 @@ def write_manifest( repository: str = "llama3", tag: str = "1b", digest: str = DIGEST_A, - size: int = 1234, + size: int = 12, content: object | None = None, raw_text: str | None = None, ) -> Path: diff --git a/tests/test_ollama_adapter.py b/tests/test_ollama_adapter.py index f8a4eb7..56376bd 100644 --- a/tests/test_ollama_adapter.py +++ b/tests/test_ollama_adapter.py @@ -29,7 +29,7 @@ def test_finds_ready_model(self, make_config): assert model.readiness is ModelReadiness.READY assert model.blob_path == config.ollama_blobs_dir / blob_filename(DIGEST_A) assert model.blob_size == 12 # 4 header bytes + 8 padding - assert model.declared_size == 1234 + assert model.declared_size == 12 assert model.issues == () def test_declared_size_garbage_is_tolerated(self, make_config): @@ -191,3 +191,38 @@ def test_multiple_models_sorted_by_path(self, make_config): write_manifest(config, repository="alpha", tag="1b", digest=DIGEST_A) models = OllamaAdapter(config).scan_models() assert [m.repository for m in models] == ["alpha", "zeta"] + + +class TestImportValidation: + def test_null_layers_does_not_abort_other_models(self, make_config): + config = make_config() + write_manifest(config, repository="broken", content={"layers": None}) + write_manifest(config, repository="healthy") + write_blob(config, DIGEST_A) + models = OllamaAdapter(config).scan_models() + assert len(models) == 2 + assert models[0].readiness is ModelReadiness.INVALID + assert models[1].readiness is ModelReadiness.READY + + def test_truncated_blob_is_invalid_even_with_gguf_header(self, make_config): + config = make_config() + write_manifest(config, size=10000) + write_blob(config, DIGEST_A) + assert scan_one(config).readiness is ModelReadiness.INVALID + + def test_non_utf8_manifest_does_not_abort_scan(self, make_config): + config = make_config() + path = write_manifest(config) + path.write_bytes(b"\xff") + report = OllamaAdapter(config).scan_report() + assert not report.complete + assert report.errors + + def test_digest_cannot_escape_blob_directory(self, make_config, tmp_path): + config = make_config() + outside = tmp_path / "outside.gguf" + outside.write_bytes(b"GGUF00000000") + write_manifest(config, digest=str(outside), size=12) + model = scan_one(config) + assert model.readiness is ModelReadiness.INVALID + assert model.blob_path is None From 9b1d385c5a51bdd85f1f6050fefe4a1210d9107b Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:28:37 +0530 Subject: [PATCH 02/11] fix: serialize sync and prune across processes - hold an OS-backed state lock across each sync or prune operation - release locks when a process exits and report contention clearly - tolerate invalid state encoding and test interrupted state writes Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. --- src/studiolink/service.py | 4 +++ src/studiolink/state.py | 43 ++++++++++++++++++++++- src/studiolink/syncer.py | 10 ++++++ tests/conftest.py | 18 ++++++++++ tests/test_failure_paths.py | 63 ++++++++++++++++++++++++++++++++++ tests/test_import_lifecycle.py | 18 ++++++++++ 6 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tests/test_failure_paths.py create mode 100644 tests/test_import_lifecycle.py diff --git a/src/studiolink/service.py b/src/studiolink/service.py index 55fd20d..80bfa50 100644 --- a/src/studiolink/service.py +++ b/src/studiolink/service.py @@ -116,6 +116,10 @@ def sync( return syncer.sync(selected, mode, import_mode, dry_run) def prune(self, *, dry_run: bool = False) -> PruneReport: + with self.state.lock(): + return self._prune_locked(dry_run=dry_run) + + def _prune_locked(self, *, dry_run: bool = False) -> PruneReport: """Safely remove unneeded aliases and stale sync records. Destructive work requires a complete Ollama scan, and aliases needed diff --git a/src/studiolink/state.py b/src/studiolink/state.py index 6a91922..e0b53e4 100644 --- a/src/studiolink/state.py +++ b/src/studiolink/state.py @@ -4,6 +4,7 @@ import logging import os from collections.abc import Mapping +from contextlib import contextmanager from pathlib import Path from studiolink.models import SyncRecord @@ -15,6 +16,46 @@ class StateStore: def __init__(self, path: Path) -> None: self.path = path + @contextmanager + def lock(self): + """Serialize import/prune transactions across processes; never wait forever.""" + self.path.parent.mkdir(parents=True, exist_ok=True) + lock_path = self.path.with_suffix(self.path.suffix + ".lock") + with lock_path.open("a+b") as handle: + if os.name == "nt": + import msvcrt + + handle.seek(0, 2) + if handle.tell() == 0: + handle.write(b"\0") + handle.flush() + handle.seek(0) + + def acquire(): + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + + def release(): + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + def acquire(): + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + + def release(): + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + try: + acquire() + except OSError as exc: + raise RuntimeError( + "Another StudioLink sync/prune is running; retry when it finishes" + ) from exc + try: + yield + finally: + release() + def get_record(self, name: str) -> SyncRecord | None: """Get a single sync record by model name.""" record_data = self._load_raw().get(name) @@ -74,7 +115,7 @@ def _load_raw(self) -> dict[str, object]: return {} try: payload = json.loads(self.path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: + except (json.JSONDecodeError, UnicodeError) as exc: logger.warning( "Corrupted state file at %s: %s. Starting fresh.", self.path, exc ) diff --git a/src/studiolink/syncer.py b/src/studiolink/syncer.py index 4e16a93..5badd2f 100644 --- a/src/studiolink/syncer.py +++ b/src/studiolink/syncer.py @@ -61,6 +61,16 @@ def sync( import_mode, dry_run, ) + with self.state.lock(): + return self._sync_locked(models, link_mode, import_mode, dry_run) + + def _sync_locked( + self, + models: list[OllamaModel], + link_mode: LinkMode, + import_mode: ImportMode, + dry_run: bool, + ) -> list[SyncResult]: records = self.state.get_all_records() logger.debug("Loaded %d existing sync record(s)", len(records)) results: list[SyncResult] = [] diff --git a/tests/conftest.py b/tests/conftest.py index 71194e6..2a349b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -220,3 +220,21 @@ def _make(config: StudioLinkConfig | None = None): return syncer, cfg, fake_lmstudio return _make + + +class FilesystemLMStudio(FakeLMStudio): + def __init__(self, config): + super().__init__() + self.config = config + + +@pytest.fixture +def library(make_config): + config = make_config() + adapter = FilesystemLMStudio(config) + from studiolink.service import StudioLinkService + + service = StudioLinkService(config=config, lmstudio=adapter) + manifest = write_manifest(config) + blob = write_blob(config, DIGEST_A) + return config, service, adapter, manifest, blob diff --git a/tests/test_failure_paths.py b/tests/test_failure_paths.py new file mode 100644 index 0000000..66bd5d3 --- /dev/null +++ b/tests/test_failure_paths.py @@ -0,0 +1,63 @@ +"""Adversarial checks at filesystem and process boundaries.""" + +from __future__ import annotations + +import errno +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from studiolink.state import StateStore + + +def test_interrupted_atomic_state_write_preserves_old_json(tmp_path, monkeypatch): + from conftest import make_sync_record + + state = StateStore(tmp_path / "state.json") + state.upsert(make_sync_record("old:1")) + old = state.path.read_bytes() + + def failed_replace(*args, **kwargs): + raise OSError(errno.ENOSPC, "disk full") + + with monkeypatch.context() as patch: + patch.setattr(os, "replace", failed_replace) + with pytest.raises(OSError): + state.upsert(make_sync_record("new:1")) + assert state.path.read_bytes() == old + assert set(state.get_all_records()) == {"old:1"} + state.upsert(make_sync_record("new:1")) + assert set(state.get_all_records()) == {"old:1", "new:1"} + + +def test_os_releases_lock_after_killed_process(tmp_path): + state = StateStore(tmp_path / "state.json") + code = """ +import sys, time +from pathlib import Path +from studiolink.state import StateStore +with StateStore(Path(sys.argv[1])).lock(): + print("LOCKED", flush=True) + time.sleep(30) +""" + env = os.environ.copy() + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1] / "src") + child = subprocess.Popen( + [sys.executable, "-c", code, str(state.path)], + env=env, + stdout=subprocess.PIPE, + text=True, + ) + try: + assert child.stdout.readline().strip() == "LOCKED" + with pytest.raises(RuntimeError, match="Another StudioLink"): + with state.lock(): + pytest.fail("overlapping transaction was allowed") + finally: + child.kill() + child.wait(timeout=5) + with state.lock(): + assert True diff --git a/tests/test_import_lifecycle.py b/tests/test_import_lifecycle.py new file mode 100644 index 0000000..db88ca9 --- /dev/null +++ b/tests/test_import_lifecycle.py @@ -0,0 +1,18 @@ +"""Exercise imports with real files, including the LM Studio destination.""" + +import pytest + + +def destination(config, model): + return config.lmstudio_models_dir / model.user_repo / model.import_filename + + +def test_sync_and_prune_do_not_overlap(library): + _, service, adapter, _, _ = library + with service.state.lock(): + with pytest.raises(RuntimeError, match="Another StudioLink"): + service.sync(sync_all=True) + with pytest.raises(RuntimeError, match="Another StudioLink"): + service.prune() + assert adapter.calls == [] + assert service.sync(sync_all=True)[0].status == "synced" From 5d5e625564d151ee5fb8325a1ac178bfb938c300 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:29:07 +0530 Subject: [PATCH 03/11] fix: recover interrupted LM Studio imports - stage and verify imports before publishing without overwriting files - remove partial imports on failure and recover abandoned transactions - validate LM Studio's configured destination before importing - cover disk-full errors, timeouts, interrupts, and process death Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. --- src/studiolink/config.py | 4 +- src/studiolink/files.py | 65 +++++++++++++ src/studiolink/lmstudio_adapter.py | 89 ++++++++++++------ src/studiolink/models.py | 38 +++++--- src/studiolink/syncer.py | 3 + src/studiolink/transactions.py | 144 +++++++++++++++++++++++++++++ tests/test_import_transactions.py | 126 +++++++++++++++++++++++++ tests/test_lmstudio_adapter.py | 82 +++++++++++++++- 8 files changed, 509 insertions(+), 42 deletions(-) create mode 100644 src/studiolink/files.py create mode 100644 src/studiolink/transactions.py create mode 100644 tests/test_import_transactions.py diff --git a/src/studiolink/config.py b/src/studiolink/config.py index 3b8d138..4e658cf 100644 --- a/src/studiolink/config.py +++ b/src/studiolink/config.py @@ -36,7 +36,7 @@ def _default_lms_exe(home: Path) -> Path: return Path(shutil.which("lms") or home / ".lmstudio" / "bin" / "lms") -def _default_lmstudio_models_dir(home: Path) -> Path: +def lmstudio_models_directory(home: Path) -> Path: fallback = home / ".lmstudio" / "models" settings_path = home / ".lmstudio" / "settings.json" try: @@ -88,7 +88,7 @@ def from_env(cls) -> "StudioLinkConfig": ollama_blobs_dir=ollama_models_dir / "blobs", lmstudio_models_dir=_expand_path( "STUDIOLINK_LMSTUDIO_MODELS_DIR", - _default_lmstudio_models_dir(home), + lmstudio_models_directory(home), ), state_dir=state_dir, state_file=state_dir / "state.json", diff --git a/src/studiolink/files.py b/src/studiolink/files.py new file mode 100644 index 0000000..eb8fba2 --- /dev/null +++ b/src/studiolink/files.py @@ -0,0 +1,65 @@ +"""Verify and remove only files belonging to a tracked import.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from studiolink.models import LinkMode, SyncRecord + + +def same_file(left: Path, right: Path) -> bool: + try: + return left.samefile(right) + except OSError: + return False + + +def identity(path: Path) -> tuple[int, int, int, int]: + stat = path.lstat() + return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns + + +def equal_contents(left: Path, right: Path) -> bool: + """Compare without allocating model-sized buffers or writing more copies.""" + try: + if left.stat().st_size != right.stat().st_size: + return False + with left.open("rb") as a, right.open("rb") as b: + while chunk := a.read(1024 * 1024): + if chunk != b.read(len(chunk)): + return False + return not b.read(1) + except OSError: + return False + + +def destination_path(root: Path, user_repo: str, filename: str) -> Path: + parts = user_repo.split("/") + if len(parts) != 2 or any(p in ("", ".", "..") or "\\" in p for p in parts): + raise ValueError(f"Invalid import repository: {user_repo}") + return root.absolute().joinpath(*parts, Path(filename).name) + + +def healthy_destination( + path: Path, blob: Path, mode: LinkMode, record: SyncRecord | None = None +) -> bool: + from studiolink.models import LinkMode + + try: + if not path.is_file(): + return False + if mode is LinkMode.HARD_LINK: + return not path.is_symlink() and same_file(path, blob) + if mode is LinkMode.SYMBOLIC_LINK: + return path.is_symlink() and same_file(path, blob) + if path.is_symlink() or same_file(path, blob): + return False + if path.stat().st_size != blob.stat().st_size: + return False + if record is not None and record.imported_identity is not None: + return identity(path) == record.imported_identity + return equal_contents(path, blob) + except OSError: + return False diff --git a/src/studiolink/lmstudio_adapter.py b/src/studiolink/lmstudio_adapter.py index 216534f..ab383a5 100644 --- a/src/studiolink/lmstudio_adapter.py +++ b/src/studiolink/lmstudio_adapter.py @@ -1,9 +1,12 @@ from __future__ import annotations import subprocess +from pathlib import Path -from studiolink.config import StudioLinkConfig +from studiolink.config import StudioLinkConfig, lmstudio_models_directory +from studiolink.files import destination_path, healthy_destination from studiolink.models import ImportResult, LinkMode +from studiolink.transactions import TRANSACTIONS, import_transaction, publish_import DEFAULT_TIMEOUT = 60 # Importing with --copy moves multi-GB GGUF files; give it an hour instead of @@ -70,34 +73,64 @@ def import_model( link_mode: LinkMode, dry_run: bool = False, ) -> ImportResult: - command = [ - str(self.config.lms_exe), - "import", - source_path, - "--yes", - "--user-repo", - user_repo, - link_mode.lms_flag, - ] - if dry_run: - command.append("--dry-run") - - try: - result = self._run(command, check=True, timeout=IMPORT_TIMEOUT) - except subprocess.CalledProcessError as exc: + actual_directory = lmstudio_models_directory(Path.home()) + if actual_directory.resolve() != self.config.lmstudio_models_dir.resolve(): raise LMStudioError( - f"LM Studio import failed for {source_path}: {exc.stderr}", - exc.cmd, - exc.returncode, - exc.stderr, - ) from exc - return ImportResult( - command=tuple(command), - stdout=result.stdout, - stderr=result.stderr, - return_code=result.returncode, - dry_run=dry_run, + "StudioLink's models directory differs from LM Studio's models directory " + f"({actual_directory}); update LM Studio's models setting or " + "STUDIOLINK_LMSTUDIO_MODELS_DIR before importing", + [], + -1, + "models directory mismatch", + ) + source = Path(source_path).absolute() + target = destination_path( + self.config.lmstudio_models_dir, user_repo, source.name ) + if target.exists() or target.is_symlink(): + raise LMStudioError( + f"Import destination already exists: {target}", + [], + -1, + "destination exists", + ) + with import_transaction(self.config.lmstudio_models_dir, source.name) as staged: + command = [ + str(self.config.lms_exe), + "import", + str(source), + "--yes", + "--user-repo", + f"{TRANSACTIONS}/{staged.parent.name}", + link_mode.lms_flag, + ] + if dry_run: + command.append("--dry-run") + try: + result = self._run(command, check=True, timeout=IMPORT_TIMEOUT) + except subprocess.CalledProcessError as exc: + raise LMStudioError( + f"LM Studio import failed for {source_path}: {exc.stderr or exc.stdout}", + exc.cmd, + exc.returncode, + exc.stderr, + ) from exc + if not dry_run: + if not healthy_destination(staged, source, link_mode): + raise LMStudioError( + f"LM Studio produced an invalid {link_mode} import", + command, + -1, + "invalid imported file", + ) + publish_import(staged, target) + return ImportResult( + command=tuple(command), + stdout=f"Imported model to {target}" if not dry_run else result.stdout, + stderr=result.stderr, + return_code=result.returncode, + dry_run=dry_run, + ) @staticmethod def _run( @@ -122,7 +155,7 @@ def _run( ) from exc except OSError as exc: raise LMStudioError( - f"Could not execute LM Studio CLI: {exc}", + f"Could not run LM Studio CLI {command[0]}: {exc}", command, -1, str(exc), diff --git a/src/studiolink/models.py b/src/studiolink/models.py index f8b5b04..f4c4e63 100644 --- a/src/studiolink/models.py +++ b/src/studiolink/models.py @@ -107,8 +107,9 @@ class SyncRecord: user_repo: str link_mode: LinkMode imported_at: datetime - imported_model_path: Path | None = None import_command: tuple[str, ...] = field(default_factory=tuple) + imported_model_path: Path | None = None + imported_identity: tuple[int, int, int, int] | None = None def to_json(self) -> dict[str, object]: return { @@ -119,12 +120,13 @@ def to_json(self) -> dict[str, object]: "user_repo": self.user_repo, "link_mode": self.link_mode.value, "imported_at": self.imported_at.isoformat(), - "imported_model_path": ( - None - if self.imported_model_path is None - else str(self.imported_model_path) - ), "import_command": list(self.import_command), + "imported_model_path": str(self.imported_model_path) + if self.imported_model_path + else None, + "imported_identity": list(self.imported_identity) + if self.imported_identity + else None, } @classmethod @@ -132,6 +134,21 @@ def from_json(cls, payload: dict[str, object]) -> "SyncRecord": import_command = payload.get("import_command", []) if not isinstance(import_command, list): raise TypeError("import_command must be a list") + raw_identity = payload.get("imported_identity") + imported_identity = None + if raw_identity is not None: + if ( + not isinstance(raw_identity, list) + or len(raw_identity) != 4 + or not all(isinstance(value, int) for value in raw_identity) + ): + raise TypeError("imported_identity must contain four integers") + imported_identity = ( + raw_identity[0], + raw_identity[1], + raw_identity[2], + raw_identity[3], + ) return cls( canonical_name=str(payload["canonical_name"]), digest=str(payload["digest"]), @@ -140,11 +157,10 @@ def from_json(cls, payload: dict[str, object]) -> "SyncRecord": user_repo=str(payload["user_repo"]), link_mode=LinkMode(str(payload["link_mode"])), imported_at=datetime.fromisoformat(str(payload["imported_at"])), - imported_model_path=( - Path(str(payload["imported_model_path"])) - if payload.get("imported_model_path") - else None - ), + imported_model_path=Path(str(payload["imported_model_path"])) + if payload.get("imported_model_path") + else None, + imported_identity=imported_identity, import_command=tuple(str(item) for item in import_command), ) diff --git a/src/studiolink/syncer.py b/src/studiolink/syncer.py index 5badd2f..1eaa490 100644 --- a/src/studiolink/syncer.py +++ b/src/studiolink/syncer.py @@ -20,6 +20,7 @@ ) from studiolink.ports import LMStudioPort from studiolink.state import StateStore +from studiolink.transactions import recover_imports logger = logging.getLogger("studiolink") @@ -71,6 +72,8 @@ def _sync_locked( import_mode: ImportMode, dry_run: bool, ) -> list[SyncResult]: + if not dry_run: + recover_imports(self.config.lmstudio_models_dir) records = self.state.get_all_records() logger.debug("Loaded %d existing sync record(s)", len(records)) results: list[SyncResult] = [] diff --git a/src/studiolink/transactions.py b/src/studiolink/transactions.py new file mode 100644 index 0000000..7cfee64 --- /dev/null +++ b/src/studiolink/transactions.py @@ -0,0 +1,144 @@ +"""Recoverable temporary destinations for imports and atomic replacements.""" + +from __future__ import annotations + +import ctypes +import errno +import json +import os +import sys +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +from studiolink.models import PruneResult +from studiolink.state import StateStore + +TRANSACTIONS = ".studiolink-transactions" +MARKER = "owner.json" + + +def _root(models: Path) -> Path: + root = models / TRANSACTIONS + if root.is_symlink(): + raise RuntimeError(f"Transaction directory must not be a symlink: {root}") + return root + + +def _cleanup(work: Path, filename: str) -> None: + (work / filename).unlink(missing_ok=True) + (work / MARKER).unlink(missing_ok=True) + + +def _remove_empty(work: Path) -> None: + (work / "lock.json.lock").unlink(missing_ok=True) + work.rmdir() + + +def recover_imports(models: Path, *, dry_run: bool = False) -> tuple[PruneResult, ...]: + root = _root(models) + if not root.exists(): + return () + results: list[PruneResult] = [] + for work in root.iterdir(): + if ( + work.is_symlink() + or not work.is_dir() + or not work.name.startswith("import-") + ): + continue + marker = work / MARKER + try: + payload = json.loads(marker.read_text(encoding="utf-8")) + except (FileNotFoundError, ValueError): + continue + filename = payload.get("filename") if isinstance(payload, dict) else None + if ( + not isinstance(filename, str) + or Path(filename).name != filename + or filename in ("", ".", "..", MARKER, "lock.json.lock") + ): + continue + try: + with StateStore(work / "lock.json").lock(): + target = work / filename + try: + stat = target.lstat() + except FileNotFoundError: + stat = None + is_symlink = target.is_symlink() + if not dry_run: + _cleanup(work, filename) + if stat is not None: + results.append( + PruneResult( + target, + stat.st_size, + not is_symlink and stat.st_nlink == 1, + "abandoned import transaction", + not dry_run, + ) + ) + except RuntimeError: + # Another process may still be importing into this directory. + continue + if not dry_run: + _remove_empty(work) + return tuple(results) + + +@contextmanager +def import_transaction(models: Path, filename: str) -> Iterator[Path]: + root = _root(models) + root.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix="import-", dir=root)) + try: + with StateStore(work / "lock.json").lock(): + (work / MARKER).write_text( + json.dumps({"filename": filename}), encoding="utf-8" + ) + try: + yield work / filename + finally: + _cleanup(work, filename) + finally: + # Never recursively remove a directory containing unknown files. + if not (work / MARKER).exists(): + _remove_empty(work) + + +def publish_import(staged: Path, target: Path) -> None: + """Publish without overwriting a file created by another importer.""" + target.parent.mkdir(parents=True, exist_ok=True) + if staged.is_symlink(): + target.symlink_to(os.readlink(staged)) + else: + # Both paths are on the LM Studio filesystem; no second weight copy. + try: + os.link(staged, target) + except OSError as exc: + if exc.errno not in (errno.EPERM, errno.ENOTSUP): + raise + _rename_without_overwrite(staged, target) + staged.unlink(missing_ok=True) + + +def _rename_without_overwrite(source: Path, target: Path) -> None: + # exFAT and some network filesystems support rename but not hard links. + if os.name == "nt": + os.rename(source, target) # Windows rename already refuses replacement. + return + libc = ctypes.CDLL(None, use_errno=True) + if sys.platform.startswith("linux") and hasattr(libc, "renameat2"): + result = libc.renameat2(-100, os.fsencode(source), -100, os.fsencode(target), 1) + elif sys.platform == "darwin" and hasattr(libc, "renamex_np"): + result = libc.renamex_np(os.fsencode(source), os.fsencode(target), 4) + else: + raise OSError( + errno.ENOTSUP, + "Filesystem cannot publish an import without overwriting existing files", + ) + if result != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), str(target)) diff --git a/tests/test_import_transactions.py b/tests/test_import_transactions.py new file mode 100644 index 0000000..c1eb6d7 --- /dev/null +++ b/tests/test_import_transactions.py @@ -0,0 +1,126 @@ +"""Failures in the actual adapter must never leave a permanent partial model.""" + +from __future__ import annotations + +import errno +import shutil +import subprocess +from pathlib import Path + +import pytest +from conftest import DIGEST_A, write_blob, write_manifest + +import studiolink.lmstudio_adapter as adapter_module +from studiolink.models import LinkMode +from studiolink.service import StudioLinkService + + +@pytest.mark.parametrize("failure", ["disk-full", "timeout", "interrupt"]) +def test_partial_cli_copy_is_removed_and_retry_works(make_config, monkeypatch, failure): + config = make_config() + monkeypatch.setattr( + adapter_module, + "lmstudio_models_directory", + lambda home: config.lmstudio_models_dir, + ) + service = StudioLinkService(config=config) + write_manifest(config) + blob = write_blob(config, DIGEST_A) + calls = [] + + def run(command, **kwargs): + calls.append(command) + source = Path(command[2]) + target = ( + config.lmstudio_models_dir + / command[command.index("--user-repo") + 1] + / source.name + ) + target.parent.mkdir(parents=True, exist_ok=True) + if len(calls) == 1: + target.write_bytes(source.read_bytes()[:5]) + if failure == "timeout": + raise subprocess.TimeoutExpired(command, 3600) + if failure == "interrupt": + raise KeyboardInterrupt + raise subprocess.CalledProcessError(1, command, stderr="disk full") + shutil.copyfile(source, target) + return subprocess.CompletedProcess(command, 0, stdout="copied", stderr="") + + monkeypatch.setattr(subprocess, "run", run) + if failure == "interrupt": + with pytest.raises(KeyboardInterrupt): + service.sync(sync_all=True, link_mode=LinkMode.COPY) + else: + assert service.sync(sync_all=True, link_mode=LinkMode.COPY)[0].status == "error" + assert not list(config.lmstudio_models_dir.rglob("*.gguf")) + assert blob.stat().st_nlink == 1 + assert service.sync(sync_all=True, link_mode=LinkMode.COPY)[0].status == "synced" + assert len(list(config.lmstudio_models_dir.rglob("*.gguf"))) == 1 + + +def test_killed_process_leaves_recoverable_temporary_import(tmp_path): + import os + import sys + + from studiolink.transactions import recover_imports + + env = os.environ.copy() + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1] / "src") + code = """ +import os, sys +from pathlib import Path +from studiolink.transactions import import_transaction +with import_transaction(Path(sys.argv[1]), 'weights.gguf') as target: + target.write_bytes(b'GGUFpartial') + os._exit(19) +""" + result = subprocess.run( + [sys.executable, "-c", code, str(tmp_path)], env=env, timeout=10 + ) + assert result.returncode == 19 + assert len(list(tmp_path.rglob("*.gguf"))) == 1 + recover_imports(tmp_path) + assert not list(tmp_path.rglob("*.gguf")) + assert not list((tmp_path / ".studiolink-transactions").iterdir()) + + +def test_recovery_preserves_active_transaction(tmp_path): + from studiolink.transactions import import_transaction, recover_imports + + with import_transaction(tmp_path, "weights.gguf") as target: + target.write_bytes(b"GGUFpartial") + recover_imports(tmp_path) + assert target.is_file() + assert not target.exists() + + +def test_publication_never_overwrites_existing_destination(tmp_path): + from studiolink.transactions import publish_import + + staged = tmp_path / "staged" + staged.write_bytes(b"GGUFnew") + target = tmp_path / "existing" + target.write_bytes(b"user data") + with pytest.raises(FileExistsError): + publish_import(staged, target) + assert target.read_bytes() == b"user data" + assert staged.read_bytes() == b"GGUFnew" + + +def test_copy_publication_works_without_filesystem_hardlinks(tmp_path, monkeypatch): + import os + + from studiolink.transactions import publish_import + + staged = tmp_path / "staged" + staged.write_bytes(b"GGUFnew") + target = tmp_path / "model" + + def unsupported(*args, **kwargs): + raise OSError(errno.EOPNOTSUPP, "hardlinks unsupported") + + monkeypatch.setattr(os, "link", unsupported) + publish_import(staged, target) + assert target.read_bytes() == b"GGUFnew" + assert not staged.exists() diff --git a/tests/test_lmstudio_adapter.py b/tests/test_lmstudio_adapter.py index 8b9858c..1ffcf16 100644 --- a/tests/test_lmstudio_adapter.py +++ b/tests/test_lmstudio_adapter.py @@ -1,12 +1,92 @@ +import subprocess +from pathlib import Path + import pytest +import studiolink.lmstudio_adapter as adapter_module from studiolink.lmstudio_adapter import LMStudioAdapter, LMStudioError from studiolink.models import LinkMode +def test_os_error_has_actionable_cli_message(make_config, monkeypatch): + def missing(*args, **kwargs): + raise FileNotFoundError("missing lms binary") + + monkeypatch.setattr(subprocess, "run", missing) + with pytest.raises(LMStudioError, match="missing lms binary"): + LMStudioAdapter(make_config()).get_version() + + +def test_import_failure_includes_stdout(make_config, monkeypatch): + def failure(*args, **kwargs): + raise subprocess.CalledProcessError( + 1, args[0], output="Target file already exists", stderr="" + ) + + monkeypatch.setattr(subprocess, "run", failure) + config = make_config() + monkeypatch.setattr( + adapter_module, + "lmstudio_models_directory", + lambda home: config.lmstudio_models_dir, + ) + with pytest.raises(LMStudioError, match="Target file already exists"): + LMStudioAdapter(config).import_model( + "model.gguf", user_repo="ollama/test", link_mode=LinkMode.HARD_LINK + ) + + +@pytest.mark.parametrize("mode", list(LinkMode)) +def test_import_command_preserves_requested_mode(make_config, monkeypatch, mode): + config = make_config() + monkeypatch.setattr( + adapter_module, + "lmstudio_models_directory", + lambda home: config.lmstudio_models_dir, + ) + commands = [] + + def run(command, **kwargs): + commands.append((command, kwargs)) + return subprocess.CompletedProcess(command, 0, stdout="ok", stderr="") + + monkeypatch.setattr(subprocess, "run", run) + LMStudioAdapter(config).import_model( + "a model.gguf", user_repo="ollama/test", link_mode=mode, dry_run=True + ) + command, options = commands[0] + assert command[:2] == [str(config.lms_exe), "import"] + assert Path(command[2]).name == "a model.gguf" + assert command[3:5] == ["--yes", "--user-repo"] + assert command[5].startswith(".studiolink-transactions/import-") + assert command[6:] == [mode.lms_flag, "--dry-run"] + assert options["timeout"] == adapter_module.IMPORT_TIMEOUT + + +def test_mismatched_directory_refused_before_import(make_config, monkeypatch, tmp_path): + config = make_config() + monkeypatch.setattr( + adapter_module, "lmstudio_models_directory", lambda home: tmp_path / "elsewhere" + ) + + def must_not_run(*args, **kwargs): + pytest.fail("must not import into the wrong directory") + + monkeypatch.setattr(subprocess, "run", must_not_run) + with pytest.raises(LMStudioError, match="models directory differs"): + LMStudioAdapter(config).import_model( + "model.gguf", user_repo="ollama/test", link_mode=LinkMode.COPY + ) + + def test_import_wraps_os_execution_error(make_config, monkeypatch): config = make_config() adapter = LMStudioAdapter(config) + monkeypatch.setattr( + adapter_module, + "lmstudio_models_directory", + lambda home: config.lmstudio_models_dir, + ) def missing_executable(*args, **kwargs): raise FileNotFoundError("lms is missing") @@ -15,7 +95,7 @@ def missing_executable(*args, **kwargs): "studiolink.lmstudio_adapter.subprocess.run", missing_executable ) - with pytest.raises(LMStudioError, match="Could not execute LM Studio CLI"): + with pytest.raises(LMStudioError, match="Could not run LM Studio CLI"): adapter.import_model( "model.gguf", user_repo="ollama/model", From 3469d30ff5683bd14de6645997dd5410e263b72b Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:29:37 +0530 Subject: [PATCH 04/11] fix: verify and repair imported model files - verify destination identity and link mode before skipping a sync - repair missing imports and recover lost state across mode changes - replace aliases and managed targets atomically; keep dry runs read-only - cover moved libraries, shared direct targets, and cross-volume staging Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. --- src/studiolink/files.py | 37 +++++ src/studiolink/models.py | 30 +++- src/studiolink/service.py | 16 +- src/studiolink/syncer.py | 265 ++++++++++++++++++++++----------- tests/conftest.py | 17 ++- tests/test_failure_paths.py | 55 +++++++ tests/test_import_lifecycle.py | 168 +++++++++++++++++++++ tests/test_models.py | 14 +- tests/test_service.py | 16 +- tests/test_syncer.py | 25 +++- 10 files changed, 518 insertions(+), 125 deletions(-) diff --git a/src/studiolink/files.py b/src/studiolink/files.py index eb8fba2..07ea94d 100644 --- a/src/studiolink/files.py +++ b/src/studiolink/files.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from pathlib import Path from typing import TYPE_CHECKING @@ -42,6 +43,42 @@ def destination_path(root: Path, user_repo: str, filename: str) -> Path: return root.absolute().joinpath(*parts, Path(filename).name) +def record_destination(record: SyncRecord, root: Path) -> Path: + path = record.imported_model_path or destination_path( + root, record.user_repo, record.import_alias_path.name + ) + # Resolve the parent only: the imported file itself may be a symlink to Ollama. + if not path.parent.resolve().is_relative_to(root.resolve()): + # LM Studio's library may have moved since the record was saved. Check + # its expected location under the new root; never delete the old path. + path = destination_path(root, record.user_repo, record.import_alias_path.name) + if not path.parent.resolve().is_relative_to(root.resolve()): + raise ValueError( + f"Import directory resolves outside the configured models directory: {path}" + ) + return path + + +def owns_destination(record: SyncRecord, path: Path) -> bool: + from studiolink.models import LinkMode + + try: + if record.imported_identity is not None: + return identity(path) == record.imported_identity + # Legacy records have no file identity. Require evidence before removal. + if path.is_symlink(): + target = path.parent / os.readlink(path) + return target.absolute() == record.import_alias_path.absolute() + for source in (record.import_alias_path, record.blob_path): + if same_file(path, source): + return True + if record.link_mode is LinkMode.COPY and equal_contents(path, source): + return True + except OSError: + pass + return False + + def healthy_destination( path: Path, blob: Path, mode: LinkMode, record: SyncRecord | None = None ) -> bool: diff --git a/src/studiolink/models.py b/src/studiolink/models.py index f4c4e63..3e4eee8 100644 --- a/src/studiolink/models.py +++ b/src/studiolink/models.py @@ -166,16 +166,30 @@ def from_json(cls, payload: dict[str, object]) -> "SyncRecord": def is_synced( - model: OllamaModel, - record: SyncRecord | None, - *, - expected_target: Path | None = None, + model: OllamaModel, record: SyncRecord | None, models_dir: Path | None = None ) -> bool: - """Check if state and the imported LM Studio file match the model.""" - if record is None or record.digest != model.model_digest: + """A saved digest is insufficient: the imported file must still work.""" + from studiolink.files import healthy_destination, record_destination + + if ( + record is None + or record.digest != model.model_digest + or model.readiness is not ModelReadiness.READY + or model.blob_path is None + or not model.blob_present + ): + return False + try: + target = ( + record_destination(record, models_dir) + if models_dir + else record.imported_model_path + ) + return target is not None and healthy_destination( + target, model.blob_path, record.link_mode, record + ) + except (OSError, ValueError): return False - target = record.imported_model_path or expected_target - return target.is_file() if target is not None else True @dataclass(slots=True, frozen=True) diff --git a/src/studiolink/service.py b/src/studiolink/service.py index 80bfa50..d09f88d 100644 --- a/src/studiolink/service.py +++ b/src/studiolink/service.py @@ -77,7 +77,11 @@ def status(self) -> list[StatusEntry]: return [ StatusEntry( model=model, - synced=self._record_is_synced(model, records.get(model.canonical_name)), + synced=is_synced( + model, + records.get(model.canonical_name), + self.config.lmstudio_models_dir, + ), sync_record=records.get(model.canonical_name), ) for model in models @@ -351,16 +355,6 @@ def _volume_compatibility_check(self) -> DoctorCheck: ) return DoctorCheck("hard-link volume compatibility", ok, details) - def _record_is_synced(self, model: OllamaModel, record: SyncRecord | None) -> bool: - expected_target = None - if record is not None and record.imported_model_path is None: - expected_target = ( - self.config.lmstudio_models_dir - / record.user_repo - / record.import_alias_path.name - ) - return is_synced(model, record, expected_target=expected_target) - def _select_models( self, discovered: list[OllamaModel], requested_names: list[str] ) -> list[OllamaModel]: diff --git a/src/studiolink/syncer.py b/src/studiolink/syncer.py index 1eaa490..e7389f1 100644 --- a/src/studiolink/syncer.py +++ b/src/studiolink/syncer.py @@ -1,12 +1,24 @@ from __future__ import annotations +import errno import logging import os +import shutil import subprocess +import tempfile +from dataclasses import replace from datetime import datetime, timezone from pathlib import Path from studiolink.config import StudioLinkConfig +from studiolink.files import ( + destination_path, + healthy_destination, + identity, + owns_destination, + record_destination, + same_file, +) from studiolink.lmstudio_adapter import LMStudioError from studiolink.models import ( ImportMode, @@ -20,23 +32,11 @@ ) from studiolink.ports import LMStudioPort from studiolink.state import StateStore -from studiolink.transactions import recover_imports +from studiolink.transactions import import_transaction, recover_imports logger = logging.getLogger("studiolink") -def same_file(left: Path, right: Path) -> bool: - """Check whether two paths reference the same inode (hard link).""" - try: - left_stat, right_stat = left.stat(), right.stat() - except OSError: - return False - return (left_stat.st_dev, left_stat.st_ino) == ( - right_stat.st_dev, - right_stat.st_ino, - ) - - class Syncer: def __init__( self, @@ -78,9 +78,16 @@ def _sync_locked( logger.debug("Loaded %d existing sync record(s)", len(records)) results: list[SyncResult] = [] for model in models: - result = self._sync_one(model, records, link_mode, import_mode, dry_run) + try: + result = self._sync_one(model, records, link_mode, import_mode, dry_run) + except (OSError, RuntimeError, ValueError) as exc: + result = SyncResult(model=model, status="error", message=str(exc)) results.append(result) - if result.record and not dry_run: + if ( + result.record + and not dry_run + and result.record != records.get(model.canonical_name) + ): records[result.record.canonical_name] = result.record self.state.save(records) logger.debug("Saved sync record for %s", model.canonical_name) @@ -100,20 +107,25 @@ def _sync_one( model.readiness, ) existing = records.get(model.canonical_name) - expected_target = None - if existing is not None and existing.imported_model_path is None: - expected_target = ( - self.config.lmstudio_models_dir - / existing.user_repo - / existing.import_alias_path.name - ) - if is_synced(model, existing, expected_target=expected_target): + if ( + existing is not None + and is_synced(model, existing, self.config.lmstudio_models_dir) + and existing.link_mode is link_mode + ): logger.debug("Model %s is already synced, skipping", model.canonical_name) + verified_target = record_destination( + existing, self.config.lmstudio_models_dir + ) return SyncResult( model=model, status="skipped", - message="already synced according to StudioLink state", - record=existing, + message="imported file verified; already synced", + record=replace( + existing, + imported_model_path=verified_target, + imported_identity=identity(verified_target), + blob_path=model.blob_path, + ), ) if model.readiness is ModelReadiness.STALE: logger.debug("Model %s is stale (blob missing)", model.canonical_name) @@ -137,6 +149,20 @@ def _sync_one( message="model is not ready for import: " + "; ".join(model.issues or ("unknown error",)), ) + if dry_run: + source = ( + model.blob_path + if import_mode is ImportMode.DIRECT + else self.config.import_staging_dir / model.import_filename + ) + target = destination_path( + self.config.lmstudio_models_dir, model.user_repo, source.name + ) + return SyncResult( + model=model, + status="dry-run", + message=f"Would sync using {link_mode}: {target}", + ) if import_mode is ImportMode.DIRECT: alias_path = model.blob_path alias_created = False @@ -158,45 +184,86 @@ def _sync_one( message=str(exc), ) - import_error: Exception | None = None - import_result: ImportResult | None = None + target = destination_path( + self.config.lmstudio_models_dir, model.user_repo, alias_path.name + ) + import_result = ImportResult( + command=(), stdout="existing import verified", stderr="", return_code=0 + ) + target_record = ( + existing + if existing + and existing.digest == model.model_digest + and existing.link_mode is link_mode + and record_destination(existing, self.config.lmstudio_models_dir) == target + else None + ) + success = False try: - import_result = self.lmstudio.import_model( - str(alias_path), - user_repo=model.user_repo, - link_mode=link_mode, - dry_run=dry_run, - ) - logger.debug("LM Studio import completed: %s", import_result.return_code) - except (LMStudioError, subprocess.TimeoutExpired) as exc: - # One failing model must not abort the rest of the batch. - logger.debug("LM Studio import failed: %s", exc) - import_error = exc - finally: - should_remove_alias = dry_run or link_mode is LinkMode.COPY - if ( - should_remove_alias - and (alias_created or link_mode is LinkMode.COPY) - and alias_path.exists() - and import_mode is not ImportMode.DIRECT + if not healthy_destination( + target, model.blob_path, link_mode, target_record ): - logger.debug("Cleaning up temporary import alias: %s", alias_path) - try: - alias_path.unlink() - alias_path.parent.rmdir() - except OSError as exc: - logger.warning( - "Could not clean up import alias %s: %s", alias_path, exc + if target.exists() or target.is_symlink(): + managed = existing is not None and ( + record_destination(existing, self.config.lmstudio_models_dir) + == target + and owns_destination(existing, target) ) - - if import_error is not None: - return SyncResult(model=model, status="error", message=str(import_error)) - if import_result is None: - return SyncResult( - model=model, - status="error", - message="LM Studio import returned no result", - ) + # Lost state can be rebuilt from a verified import even + # when the requested mode differs from its current mode. + recoverable = existing is None and any( + healthy_destination(target, model.blob_path, candidate) + for candidate in LinkMode + ) + if not (managed or recoverable): + raise RuntimeError( + f"Import destination already exists and is not verified: {target}" + ) + # Build beside the destination and replace atomically. A + # failed conversion leaves the old import usable. + self._replace_destination( + alias_path, target, link_mode, model, records + ) + import_result = ImportResult( + command=(), + stdout=f"Replaced managed import: {target}", + stderr="", + return_code=0, + ) + else: + import_result = self.lmstudio.import_model( + str(alias_path.absolute()), + user_repo=model.user_repo, + link_mode=link_mode, + dry_run=False, + ) + if not healthy_destination(target, model.blob_path, link_mode): + raise RuntimeError( + f"LM Studio did not create the expected {link_mode} import at {target}; " + "check that STUDIOLINK_LMSTUDIO_MODELS_DIR matches LM Studio's models directory" + ) + success = True + except ( + LMStudioError, + subprocess.TimeoutExpired, + OSError, + RuntimeError, + ValueError, + ) as exc: + return SyncResult(model=model, status="error", message=str(exc)) + finally: + if import_mode is not ImportMode.DIRECT and ( + (alias_created and not success) or link_mode is LinkMode.COPY + ): + # Preserve a symlink target if import succeeded but a later cleanup + # failed. A subsequent sync can adopt that verified destination. + if not (target.is_symlink() and same_file(target, alias_path)): + alias_path.unlink(missing_ok=True) + if link_mode is LinkMode.COPY: + try: + alias_path.parent.rmdir() + except OSError: + pass record = SyncRecord( canonical_name=model.canonical_name, @@ -206,49 +273,81 @@ def _sync_one( user_repo=model.user_repo, link_mode=link_mode, imported_at=datetime.now(tz=timezone.utc), - imported_model_path=( - self.config.lmstudio_models_dir / model.user_repo / alias_path.name - ), import_command=import_result.command, + imported_model_path=target, + imported_identity=identity(target), ) return SyncResult( model=model, - status="dry-run" if dry_run else "synced", + status="synced", message=_format_import_message(import_result), record=record, ) + def _replace_destination( + self, + source: Path, + target: Path, + mode: LinkMode, + model: OllamaModel, + records: dict[str, SyncRecord], + ) -> None: + with import_transaction( + self.config.lmstudio_models_dir, target.name + ) as temporary: + if mode is LinkMode.HARD_LINK: + os.link(source.resolve(), temporary) + elif mode is LinkMode.SYMBOLIC_LINK: + temporary.symlink_to(source.absolute()) + else: + shutil.copyfile(source, temporary) + os.replace(temporary, target) + def _ensure_import_alias( self, model: OllamaModel, link_mode: LinkMode ) -> tuple[Path, bool]: - staging_dir = self.config.import_staging_dir + staging = self.config.import_staging_dir if link_mode is LinkMode.COPY: - # The alias controls the human-readable LM Studio filename. Keep - # this temporary hard link on the Ollama filesystem so copy mode - # still works when StudioLink state lives on another volume. - staging_dir = self.config.ollama_models_dir / ".studiolink-imports" - return _ensure_import_alias(model, staging_dir) + staging = self.config.ollama_models_dir / ".studiolink-imports" + return _ensure_import_alias(model, staging, link_mode) -def _ensure_import_alias(model: OllamaModel, staging_dir: Path) -> tuple[Path, bool]: +def _ensure_import_alias( + model: OllamaModel, staging_dir: Path, link_mode: LinkMode = LinkMode.HARD_LINK +) -> tuple[Path, bool]: alias_path = staging_dir / model.import_filename if model.blob_path is None: raise RuntimeError(f"No blob path for model {model.canonical_name}") - if alias_path.exists(): - if same_file(alias_path, model.blob_path): - logger.debug("Import alias already exists: %s", alias_path) - return alias_path, False - # The alias survived a re-pull but now points at the old blob content; - # replace it so we never import stale weights. - logger.debug("Replacing stale import alias: %s", alias_path) - alias_path.unlink() + present = alias_path.exists() or alias_path.is_symlink() + if ( + present + and same_file(alias_path, model.blob_path) + and (link_mode is not LinkMode.HARD_LINK or not alias_path.is_symlink()) + ): + return alias_path, False staging_dir.mkdir(parents=True, exist_ok=True) + temporary = alias_path + if present: + fd, name = tempfile.mkstemp(prefix=".studiolink-", dir=staging_dir) + os.close(fd) + temporary = Path(name) + temporary.unlink() try: - os.link(model.blob_path, alias_path) - logger.debug("Created hard link: %s -> %s", alias_path, model.blob_path) + try: + os.link(model.blob_path, temporary) + except OSError as exc: + if exc.errno != errno.EXDEV or link_mode is LinkMode.HARD_LINK: + raise + # Stage across volumes without writing another full copy. + temporary.symlink_to(model.blob_path.resolve()) + if present: + os.replace(temporary, alias_path) return alias_path, True except OSError as exc: - raise RuntimeError(f"Failed to create import alias (hard link): {exc}") from exc + raise RuntimeError(f"Failed to create import alias: {exc}") from exc + finally: + if temporary != alias_path: + temporary.unlink(missing_ok=True) def _format_import_message(result: ImportResult) -> str: diff --git a/tests/conftest.py b/tests/conftest.py index 2a349b8..fac7d1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import os +import shutil import subprocess import sys from datetime import datetime, timezone @@ -157,7 +159,6 @@ def __init__(self) -> None: self.calls: list[dict] = [] self.fail_for: set[str] = set() self.timeout_for: set[str] = set() - self.models_dir: Path | None = None def get_version(self) -> str | None: return "0.3.17 (fake)" @@ -190,10 +191,16 @@ def import_model( ) if source_path in self.timeout_for: raise subprocess.TimeoutExpired(cmd=["lms"], timeout=3600) - if not dry_run and self.models_dir is not None: - target = self.models_dir / user_repo / Path(source_path).name + config = getattr(self, "config", None) + if config is not None and not dry_run: + target = config.lmstudio_models_dir / user_repo / Path(source_path).name target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(Path(source_path).read_bytes()) + if link_mode is LinkMode.HARD_LINK: + os.link(source_path, target) + elif link_mode is LinkMode.SYMBOLIC_LINK: + target.symlink_to(source_path) + else: + shutil.copyfile(source_path, target) return ImportResult( command=("lms", "import", source_path), stdout="", @@ -215,7 +222,7 @@ def make_syncer(make_config, fake_lmstudio): def _make(config: StudioLinkConfig | None = None): cfg = config or make_config() - fake_lmstudio.models_dir = cfg.lmstudio_models_dir + fake_lmstudio.config = cfg syncer = Syncer(cfg, fake_lmstudio, StateStore(cfg.state_file)) return syncer, cfg, fake_lmstudio diff --git a/tests/test_failure_paths.py b/tests/test_failure_paths.py index 66bd5d3..7ed8915 100644 --- a/tests/test_failure_paths.py +++ b/tests/test_failure_paths.py @@ -9,10 +9,29 @@ from pathlib import Path import pytest +from conftest import DIGEST_A, write_blob, write_manifest +from test_import_lifecycle import destination +from studiolink.models import LinkMode from studiolink.state import StateStore +def test_state_write_failure_can_recover_existing_copy(library, monkeypatch): + config, service, adapter, _, blob = library + + def full(*args, **kwargs): + raise OSError(errno.ENOSPC, "disk full") + + with monkeypatch.context() as patch: + patch.setattr(service.state, "_save_raw", full) + with pytest.raises(OSError, match="disk full"): + service.sync(sync_all=True, link_mode=LinkMode.COPY) + target = destination(config, service.scan()[0]) + assert target.read_bytes() == blob.read_bytes() + assert service.sync(sync_all=True, link_mode=LinkMode.COPY)[0].status == "synced" + assert len(adapter.calls) == 1 + + def test_interrupted_atomic_state_write_preserves_old_json(tmp_path, monkeypatch): from conftest import make_sync_record @@ -61,3 +80,39 @@ def test_os_releases_lock_after_killed_process(tmp_path): child.wait(timeout=5) with state.lock(): assert True + + +@pytest.mark.parametrize("mode", list(LinkMode)) +def test_real_cross_filesystem_import(make_config, fake_lmstudio, tmp_path, mode): + import tempfile + + from studiolink.service import StudioLinkService + + if ( + not Path("/dev/shm").is_dir() + or Path("/dev/shm").stat().st_dev == tmp_path.stat().st_dev + ): + pytest.skip("requires a writable second filesystem") + with tempfile.TemporaryDirectory( + prefix="studiolink-test-", dir="/dev/shm" + ) as second: + config = make_config( + lmstudio_models_dir=Path(second) / "models", + state_dir=Path(second) / "state", + state_file=Path(second) / "state/state.json", + import_staging_dir=Path(second) / "state/imports", + ) + fake_lmstudio.config = config + service = StudioLinkService(config=config, lmstudio=fake_lmstudio) + write_manifest(config) + blob = write_blob(config, DIGEST_A) + assert blob.stat().st_dev != config.state_dir.parent.stat().st_dev + result = service.sync(sync_all=True, link_mode=mode)[0] + if mode is LinkMode.HARD_LINK: + assert result.status == "error" + assert blob.stat().st_nlink == 1 + else: + assert result.status == "synced", result.message + assert ( + destination(config, service.scan()[0]).read_bytes() == blob.read_bytes() + ) diff --git a/tests/test_import_lifecycle.py b/tests/test_import_lifecycle.py index db88ca9..85d3b1f 100644 --- a/tests/test_import_lifecycle.py +++ b/tests/test_import_lifecycle.py @@ -1,12 +1,164 @@ """Exercise imports with real files, including the LM Studio destination.""" +import os +from pathlib import Path + import pytest +from conftest import DIGEST_A, write_blob, write_manifest + +from studiolink.models import ImportMode, LinkMode +from studiolink.service import StudioLinkService def destination(config, model): return config.lmstudio_models_dir / model.user_repo / model.import_filename +def test_deleted_destination_is_detected_and_repaired(library): + config, service, adapter, _, _ = library + service.sync(sync_all=True) + target = destination(config, service.scan()[0]) + target.unlink() + assert not service.status()[0].synced + assert service.sync(sync_all=True)[0].status == "synced" + assert target.is_file() + + +def test_changing_link_mode_is_not_silently_skipped(library): + config, service, _, _, blob = library + service.sync(sync_all=True, link_mode=LinkMode.COPY) + result = service.sync(sync_all=True, link_mode=LinkMode.HARD_LINK) + assert result[0].status == "synced" + assert destination(config, service.scan()[0]).samefile(blob) + + +def test_failed_import_does_not_pin_blob(library): + config, service, adapter, _, blob = library + model = service.scan()[0] + alias = config.import_staging_dir / model.import_filename + adapter.fail_for.add(str(alias)) + assert service.sync(sync_all=True)[0].status == "error" + assert not alias.exists() + assert blob.stat().st_nlink == 1 + + +def test_sync_recovers_after_state_loss_without_reimport(library): + config, service, adapter, _, _ = library + service.sync(sync_all=True) + config.state_file.unlink() + assert service.sync(sync_all=True)[0].status != "error" + assert len(adapter.calls) == 1 + assert service.state.get_all_records() + + +@pytest.mark.parametrize("first", list(LinkMode)) +@pytest.mark.parametrize("second", list(LinkMode)) +def test_state_loss_and_mode_change_recovers_verified_import(library, first, second): + config, service, _, _, blob = library + assert service.sync(sync_all=True, link_mode=first)[0].status == "synced" + config.state_file.unlink() + result = service.sync(sync_all=True, link_mode=second)[0] + assert result.status == "synced", result.message + target = destination(config, service.scan()[0]) + assert target.read_bytes() == blob.read_bytes() + assert target.is_symlink() == (second is LinkMode.SYMBOLIC_LINK) + assert target.samefile(blob) == (second is not LinkMode.COPY) + + +def test_state_loss_does_not_adopt_unrelated_file(library): + config, service, _, _, _ = library + service.sync(sync_all=True, link_mode=LinkMode.COPY) + target = destination(config, service.scan()[0]) + target.write_bytes(b"other content") + config.state_file.unlink() + assert service.sync(sync_all=True)[0].status == "error" + assert target.read_bytes() == b"other content" + + +def test_direct_tags_sharing_a_target_can_change_to_copy(library): + config, service, _, _, blob = library + write_manifest(config, tag="another") + service.sync(sync_all=True, import_mode=ImportMode.DIRECT) + results = service.sync( + sync_all=True, import_mode=ImportMode.DIRECT, link_mode=LinkMode.COPY + ) + assert all(result.status == "synced" for result in results), [ + result.message for result in results + ] + records = list(service.state.get_all_records().values()) + assert records[0].imported_model_path == records[1].imported_model_path + assert all(entry.synced for entry in service.status()) + target = records[0].imported_model_path + assert target.read_bytes() == blob.read_bytes() + assert not target.samefile(blob) + + +def test_failed_mode_conversion_preserves_working_import(library, monkeypatch): + config, service, _, _, blob = library + service.sync(sync_all=True, link_mode=LinkMode.COPY) + target = destination(config, service.scan()[0]) + before = target.read_bytes() + + def fail_link(*args, **kwargs): + raise PermissionError("denied") + + monkeypatch.setattr(os, "link", fail_link) + result = service.sync(sync_all=True, link_mode=LinkMode.HARD_LINK) + assert result[0].status == "error" + assert target.read_bytes() == before + assert not target.samefile(blob) + assert list(target.parent.glob(".studiolink-*")) == [] + + +def test_copy_can_stage_across_filesystems_without_extra_copy(library, monkeypatch): + import errno + + config, service, _, _, blob = library + original = os.link + + def cross_volume(source, target, *args, **kwargs): + if Path(target).parent == config.import_staging_dir: + raise OSError(errno.EXDEV, "cross-device link") + return original(source, target, *args, **kwargs) + + monkeypatch.setattr(os, "link", cross_volume) + assert service.sync(sync_all=True, link_mode=LinkMode.COPY)[0].status == "synced" + model = service.scan()[0] + assert not (config.import_staging_dir / model.import_filename).exists() + assert not (config.ollama_models_dir / ".studiolink-imports").exists() + assert destination(config, model).read_bytes() == blob.read_bytes() + + +def test_deleted_blob_is_not_reported_synced(library): + _, service, _, _, blob = library + service.sync(sync_all=True) + blob.unlink() + assert service.status()[0].display_status == "stale" + assert service.sync(sync_all=True)[0].status == "error" + + +def test_replaced_blob_same_digest_is_relinked(library): + config, service, _, _, blob = library + service.sync(sync_all=True) + blob.unlink() + write_blob(config, DIGEST_A) + assert not service.status()[0].synced + assert service.sync(sync_all=True)[0].status == "synced" + assert destination(config, service.scan()[0]).samefile(blob) + + +def test_dry_run_does_not_replace_existing_alias(library): + config, service, _, _, blob = library + service.sync(sync_all=True) + model = service.scan()[0] + alias = config.import_staging_dir / model.import_filename + original_inode = alias.stat().st_ino + blob.unlink() + write_blob(config, DIGEST_A) + service.sync(sync_all=True, dry_run=True) + assert alias.stat().st_ino == original_inode + + def test_sync_and_prune_do_not_overlap(library): _, service, adapter, _, _ = library with service.state.lock(): @@ -16,3 +168,19 @@ def test_sync_and_prune_do_not_overlap(library): service.prune() assert adapter.calls == [] assert service.sync(sync_all=True)[0].status == "synced" + + +@pytest.mark.parametrize("mode", [LinkMode.HARD_LINK, LinkMode.COPY]) +def test_sync_recovers_after_models_directory_is_moved(library, mode): + from dataclasses import replace + + config, service, adapter, _, _ = library + service.sync(sync_all=True, link_mode=mode) + relocated = config.lmstudio_models_dir.parent / "relocated" + config.lmstudio_models_dir.rename(relocated) + new_config = replace(config, lmstudio_models_dir=relocated) + adapter.config = new_config + new_service = StudioLinkService(config=new_config, lmstudio=adapter) + assert new_service.status()[0].synced + assert new_service.sync(sync_all=True, link_mode=mode)[0].status == "skipped" + assert len(adapter.calls) == 1 diff --git a/tests/test_models.py b/tests/test_models.py index fe9b8f1..509042a 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,3 +1,6 @@ +import os +from dataclasses import replace + from conftest import DIGEST_A, DIGEST_B, make_ollama_model, make_sync_record from studiolink.models import ( @@ -90,9 +93,14 @@ class TestIsSynced: def test_no_record(self): assert is_synced(make_ollama_model(), None) is False - def test_matching_digest(self): - model = make_ollama_model("llama3:1b", DIGEST_A) - assert is_synced(model, make_sync_record("llama3:1b", DIGEST_A)) is True + def test_matching_digest_and_live_destination(self, tmp_path): + blob = tmp_path / "blob" + blob.write_bytes(b"GGUF00000000") + target = tmp_path / "model.gguf" + os.link(blob, target) + model = make_ollama_model("llama3:1b", DIGEST_A, blob_path=blob) + record = replace(make_sync_record(), imported_model_path=target) + assert is_synced(model, record) is True def test_digest_mismatch_triggers_resync(self): model = make_ollama_model("llama3:1b", DIGEST_B) diff --git a/tests/test_service.py b/tests/test_service.py index bc49bd7..c631508 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -336,16 +336,8 @@ def test_status_reports_synced_only_when_imported_target_exists( service = build_service(config, fake_lmstudio) write_manifest(config, digest=DIGEST_A) write_blob(config, DIGEST_A) - model = OllamaAdapter(config).scan_models()[0] - target = config.lmstudio_models_dir / model.user_repo / model.import_filename - target.parent.mkdir(parents=True) - target.write_bytes(b"GGUFxxxx") - StateStore(config.state_file).upsert( - replace( - make_sync_record("llama3:1b", DIGEST_A), - imported_model_path=target, - ) - ) + fake_lmstudio.config = config + service.sync(sync_all=True) entries = service.status() @@ -358,7 +350,7 @@ def test_status_reconciles_legacy_record_with_derived_target( config = make_config() service = build_service(config, fake_lmstudio) write_manifest(config, digest=DIGEST_A) - write_blob(config, DIGEST_A) + blob = write_blob(config, DIGEST_A) record = make_sync_record("llama3:1b", DIGEST_A) target = ( config.lmstudio_models_dir @@ -366,7 +358,7 @@ def test_status_reconciles_legacy_record_with_derived_target( / record.import_alias_path.name ) target.parent.mkdir(parents=True) - target.write_bytes(b"GGUFxxxx") + os.link(blob, target) StateStore(config.state_file).upsert(record) entries = service.status() diff --git a/tests/test_syncer.py b/tests/test_syncer.py index 7cf15f9..80f5e0a 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -154,11 +154,11 @@ def test_dry_run_cleans_alias_and_skips_state(self, make_syncer): results = syncer.sync([model], **sync_kwargs(dry_run=True)) assert results[0].status == "dry-run" - assert fake.calls[0]["dry_run"] is True + assert fake.calls == [] assert not (config.import_staging_dir / model.import_filename).exists() assert StateStore(config.state_file).get_all_records() == {} - def test_dry_run_failure_still_cleans_alias(self, make_syncer): + def test_dry_run_does_not_need_working_lms(self, make_syncer): syncer, config, fake = make_syncer() write_manifest(config, digest=DIGEST_A) write_blob(config, DIGEST_A) @@ -167,7 +167,7 @@ def test_dry_run_failure_still_cleans_alias(self, make_syncer): results = syncer.sync([model], **sync_kwargs(dry_run=True)) - assert results[0].status == "error" + assert results[0].status == "dry-run" assert not (config.import_staging_dir / model.import_filename).exists() @@ -326,3 +326,22 @@ def test_sync_result_carries_model(self, make_syncer): results = syncer.sync([model], **sync_kwargs()) assert results[0].model is model assert results[0].model.readiness is ModelReadiness.READY + + +def test_failed_alias_replacement_preserves_previous_link(make_syncer, monkeypatch): + syncer, config, _ = make_syncer() + write_manifest(config) + blob = write_blob(config, DIGEST_A) + model = scan_models(config)[0] + alias, _ = _ensure_import_alias(model, config.import_staging_dir) + old_inode = alias.stat().st_ino + blob.unlink() + write_blob(config, DIGEST_A) + + def denied(*args, **kwargs): + raise PermissionError("denied") + + monkeypatch.setattr(os, "link", denied) + result = syncer.sync(scan_models(config), **sync_kwargs()) + assert result[0].status == "error" + assert alias.stat().st_ino == old_inode From d1abe313dffb5e3b681eec253d2d22e835091f84 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:30:07 +0530 Subject: [PATCH 05/11] fix: reclaim obsolete imports without breaking dependent links - remove verified old destinations and aliases after repulls and pruning - preserve untracked symbolic dependencies and changed destination files - refuse cleanup after unavailable libraries or incomplete scans - retain failed cleanup records for retry and count shared storage once - report abandoned imports and failed deletions through the CLI Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. --- src/studiolink/cli.py | 21 ++- src/studiolink/files.py | 32 +++++ src/studiolink/models.py | 2 + src/studiolink/service.py | 217 +++++++++++++++++++----------- src/studiolink/syncer.py | 29 ++++ tests/test_cli.py | 41 +++++- tests/test_failure_paths.py | 85 ++++++++++++ tests/test_import_lifecycle.py | 116 +++++++++++++++- tests/test_import_transactions.py | 20 +++ tests/test_service.py | 2 + 10 files changed, 479 insertions(+), 86 deletions(-) diff --git a/src/studiolink/cli.py b/src/studiolink/cli.py index 4119404..db4d830 100644 --- a/src/studiolink/cli.py +++ b/src/studiolink/cli.py @@ -249,26 +249,33 @@ def run_doctor(args: argparse.Namespace, service: StudioLinkService) -> int: def run_prune(args: argparse.Namespace, service: StudioLinkService) -> int: report = service.prune(dry_run=bool(args.dry_run)) + files = (*report.imports, *report.aliases) + failed = not report.dry_run and any(not item.removed for item in files) if args.json: print(json.dumps(report.to_json(), indent=2)) - return 0 + return 1 if failed else 0 verb = "Would remove" if report.dry_run else "Removed" - if not report.aliases and not report.records_removed: + if not files and not report.records_removed: print("Nothing to prune.") return 0 - for item in report.aliases: - print(f"- {verb} alias: {item.path} ({item.size} bytes) - {item.reason}") + for item in files: + action = verb if report.dry_run or item.removed else "Failed to remove" + print(f"- {action}: {item.path} ({item.size} bytes) - {item.reason}") if report.records_removed: print( f"- {verb} {len(report.records_removed)} stale sync record(s): " + ", ".join(report.records_removed) ) - would_free = sum(item.size for item in report.aliases if item.would_free) + would_free = sum( + item.size + for item in files + if item.would_free and (report.dry_run or item.removed) + ) if would_free: action = "would free" if report.dry_run else "freed" - print(f"~{would_free} bytes {action} (aliases that were the last link).") - return 0 + print(f"~{would_free} bytes {action} (files that were the last link).") + return 1 if failed else 0 def run_upgrade(args: argparse.Namespace, service: StudioLinkService) -> int: diff --git a/src/studiolink/files.py b/src/studiolink/files.py index 07ea94d..4f3994c 100644 --- a/src/studiolink/files.py +++ b/src/studiolink/files.py @@ -100,3 +100,35 @@ def healthy_destination( return equal_contents(path, blob) except OSError: return False + + +def remove_import( + record: SyncRecord, root: Path, staging: Path, *, keep: set[Path] +) -> None: + path = record_destination(record, root) + if path not in keep and (path.exists() or path.is_symlink()): + if not owns_destination(record, path): + raise RuntimeError( + f"Refusing to remove a changed or unverified import: {path}" + ) + path.unlink() + alias = record.import_alias_path + if alias.parent.resolve() == staging.resolve() and alias not in keep: + if alias.absolute() not in symlink_references(root): + alias.unlink(missing_ok=True) + + +def symlink_references(root: Path) -> dict[Path, set[Path]]: + """Find LM Studio symlinks before deleting aliases, including untracked ones.""" + references: dict[Path, set[Path]] = {} + + def fail(exc: OSError) -> None: + raise exc + + for directory, dirs, files in os.walk(root, onerror=fail): + for name in (*dirs, *files): + path = Path(directory) / name + if path.is_symlink(): + target = Path(os.path.abspath(path.parent / os.readlink(path))) + references.setdefault(target, set()).add(path) + return references diff --git a/src/studiolink/models.py b/src/studiolink/models.py index 3e4eee8..c220037 100644 --- a/src/studiolink/models.py +++ b/src/studiolink/models.py @@ -241,10 +241,12 @@ class PruneReport: dry_run: bool aliases: tuple[PruneResult, ...] = () records_removed: tuple[str, ...] = () + imports: tuple[PruneResult, ...] = () def to_json(self) -> dict[str, object]: return { "dry_run": self.dry_run, "aliases": [item.to_json() for item in self.aliases], + "imports": [item.to_json() for item in self.imports], "records_removed": list(self.records_removed), } diff --git a/src/studiolink/service.py b/src/studiolink/service.py index d09f88d..fac364d 100644 --- a/src/studiolink/service.py +++ b/src/studiolink/service.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from studiolink.config import StudioLinkConfig +from studiolink.files import owns_destination, record_destination, symlink_references from studiolink.lmstudio_adapter import LMStudioAdapter from studiolink.models import ( DoctorCheck, @@ -23,6 +24,7 @@ from studiolink.ollama_adapter import OllamaAdapter from studiolink.state import StateStore from studiolink.syncer import Syncer, same_file +from studiolink.transactions import recover_imports if TYPE_CHECKING: from studiolink.ports import LMStudioPort, OllamaPort @@ -120,105 +122,166 @@ def sync( return syncer.sync(selected, mode, import_mode, dry_run) def prune(self, *, dry_run: bool = False) -> PruneReport: + """Remove obsolete managed destinations as well as their staging aliases.""" with self.state.lock(): return self._prune_locked(dry_run=dry_run) - def _prune_locked(self, *, dry_run: bool = False) -> PruneReport: - """Safely remove unneeded aliases and stale sync records. - - Destructive work requires a complete Ollama scan, and aliases needed - by symbolic LM Studio imports are preserved. - """ + def _prune_locked(self, *, dry_run: bool) -> PruneReport: scan = self.scan_report() - if not scan.source_available: - details = "; ".join(scan.errors) or "source is unavailable" + if not scan.source_available or not self.config.ollama_blobs_dir.is_dir(): + details = "; ".join(scan.errors) or "blobs directory is unavailable" raise RuntimeError(f"Ollama library could not be scanned: {details}") if not scan.complete: details = "; ".join(scan.errors) or "unknown scan error" raise RuntimeError(f"Ollama manifest scan was incomplete: {details}") - + if not self.config.lmstudio_models_dir.is_dir(): + raise RuntimeError( + "LM Studio models directory is unavailable; refusing to prune" + ) models = list(scan.models) - expected: dict[str, OllamaModel] = { - model.import_filename: model - for model in models - if model.readiness is ModelReadiness.READY and model.blob_path is not None - } + current = {m.canonical_name: m for m in models} records = self.state.get_all_records() - unready_names = { - model.canonical_name - for model in models - if model.readiness is not ModelReadiness.READY + stale = { + name: record + for name, record in records.items() + if name not in current + or ( + current[name].readiness is ModelReadiness.READY + and record.digest != current[name].model_digest + ) } protected_aliases = { record.import_alias_path for name, record in records.items() - if record.link_mode is LinkMode.SYMBOLIC_LINK or name in unready_names + if name not in stale } - - alias_results: list[PruneResult] = [] - staging = self.config.import_staging_dir - if staging.exists(): - for path in sorted(staging.iterdir()): - if not path.is_file(): - continue - if path in protected_aliases: - logger.debug("Preserving required import alias: %s", path) - continue - model = expected.get(path.name) - if model is not None and same_file(path, model.blob_path or path): - continue - reason = ( - "alias no longer points at the current model blob" - if model is not None - else "model no longer present in Ollama (or was re-pulled)" - ) + protected_targets = { + record_destination(record, self.config.lmstudio_models_dir) + for name, record in records.items() + if name not in stale + } + recovered = recover_imports(self.config.lmstudio_models_dir, dry_run=dry_run) + references = symlink_references(self.config.lmstudio_models_dir) + expected = {m.import_filename: m for m in models} + imports: list[PruneResult] = list(recovered) + aliases: list[PruneResult] = [] + removed_names: list[str] = [] + seen: set = set() + # Simulate link-count changes in dry runs as well, so an inode shared by + # several doomed paths is counted exactly once. + removed_links: dict[tuple[int, int], int] = {} + + def remove(path, reason, results): + if results is aliases and any( + ref.is_symlink() + for ref in references.get(path.absolute(), ()) + if ref not in seen + ): + return False + if path in seen: + return True + try: + stat = path.lstat() + except FileNotFoundError: + return True + is_link = path.is_symlink() + key = (stat.st_dev, stat.st_ino) + remaining = stat.st_nlink - (removed_links.get(key, 0) if dry_run else 0) + removed = False + error = None + if not dry_run: try: - stat = path.stat() + path.unlink() + removed = True except OSError as exc: - logger.warning("Could not inspect alias %s: %s", path, exc) + error = exc + logger.warning("Could not remove %s: %s", path, exc) + if dry_run or removed: + removed_links[key] = removed_links.get(key, 0) + 1 + seen.add(path) + results.append( + PruneResult( + path=path, + size=stat.st_size, + would_free=not is_link and remaining == 1, + reason=reason if error is None else str(error), + removed=removed, + ) + ) + return dry_run or removed + + for name, record in stale.items(): + target = record_destination(record, self.config.lmstudio_models_dir) + if ( + record.link_mode is LinkMode.SYMBOLIC_LINK + and record.imported_model_path is None + ): + # A legacy record does not identify which symlink depends on this alias. + protected_aliases.add(record.import_alias_path) + continue + if target not in protected_targets and ( + target.exists() or target.is_symlink() + ): + if not owns_destination(record, target): + logger.warning("Keeping changed or unverified import: %s", target) + protected_aliases.add(record.import_alias_path) + continue + if not remove( + target, "model removed from Ollama or superseded", imports + ): + protected_aliases.add(record.import_alias_path) + continue + alias = record.import_alias_path + if ( + alias not in protected_aliases + and alias.parent.resolve() == self.config.import_staging_dir.resolve() + ): + if not remove(alias, "obsolete import alias", aliases): continue - removed = False - if not dry_run: - try: - path.unlink() - removed = True - except OSError as exc: - logger.warning("Could not remove alias %s: %s", path, exc) - alias_results.append( - PruneResult( - path=path, - size=stat.st_size, - would_free=stat.st_nlink == 1, - reason=reason, - removed=removed, + removed_names.append(name) + + for staging in ( + self.config.import_staging_dir, + self.config.ollama_models_dir / ".studiolink-imports", + ): + if staging.is_dir(): + for path in sorted(staging.iterdir()): + if ( + path in protected_aliases + or path in seen + or not (path.is_file() or path.is_symlink()) + ): + continue + model = expected.get(path.name) + # An invalid/incomplete manifest isn't proof its alias is obsolete. + if ( + staging == self.config.import_staging_dir + and model is not None + and ( + model.readiness is not ModelReadiness.READY + or ( + model.blob_path is not None + and same_file(path, model.blob_path) + ) + ) + ): + continue + reason = ( + "alias no longer points at the current model blob" + if model + else "model no longer present in Ollama (or was re-pulled)" ) - ) - - known = {model.canonical_name for model in models} - stale_names = sorted( - name - for name, record in records.items() - if name not in known and record.link_mode is not LinkMode.SYMBOLIC_LINK - ) - if stale_names and not dry_run: - remaining = { - name: record - for name, record in records.items() - if name not in stale_names - } - self.state.save(remaining) - records_removed = tuple(stale_names) + remove(path, reason, aliases) - logger.debug( - "Prune complete: %d alias(es), %d record(s) (dry_run=%s)", - len(alias_results), - len(records_removed), - dry_run, - ) + if not dry_run and removed_names: + for name in removed_names: + records.pop(name) + self.state.save(records) return PruneReport( dry_run=dry_run, - aliases=tuple(alias_results), - records_removed=records_removed, + aliases=tuple(aliases), + imports=tuple(imports), + records_removed=tuple(sorted(removed_names)), ) def doctor(self) -> list[DoctorCheck]: diff --git a/src/studiolink/syncer.py b/src/studiolink/syncer.py index e7389f1..fda3075 100644 --- a/src/studiolink/syncer.py +++ b/src/studiolink/syncer.py @@ -17,7 +17,9 @@ identity, owns_destination, record_destination, + remove_import, same_file, + symlink_references, ) from studiolink.lmstudio_adapter import LMStudioError from studiolink.models import ( @@ -242,6 +244,33 @@ def _sync_one( f"LM Studio did not create the expected {link_mode} import at {target}; " "check that STUDIOLINK_LMSTUDIO_MODELS_DIR matches LM Studio's models directory" ) + if existing is not None: + keep = {target, alias_path} + for name, record in records.items(): + if name != model.canonical_name: + keep.add(record.import_alias_path) + keep.add( + record_destination(record, self.config.lmstudio_models_dir) + ) + old_target = record_destination( + existing, self.config.lmstudio_models_dir + ) + if old_target != target: + remove_import( + existing, + self.config.lmstudio_models_dir, + self.config.import_staging_dir, + keep=keep, + ) + elif ( + existing.import_alias_path not in keep + and existing.import_alias_path.parent.resolve() + == self.config.import_staging_dir.resolve() + ): + if existing.import_alias_path.absolute() not in symlink_references( + self.config.lmstudio_models_dir + ): + existing.import_alias_path.unlink(missing_ok=True) success = True except ( LMStudioError, diff --git a/tests/test_cli.py b/tests/test_cli.py index 6198958..667696c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -220,7 +220,12 @@ def test_prune_dry_run(self, fake_service): def test_prune_json(self, fake_service, capsys): assert cli.main(["prune", "--json"]) == 0 payload = json.loads(out(capsys)) - assert payload == {"dry_run": False, "aliases": [], "records_removed": []} + assert payload == { + "dry_run": False, + "aliases": [], + "imports": [], + "records_removed": [], + } def test_prune_reports_aliases(self, fake_service, monkeypatch, capsys): from pathlib import Path @@ -333,3 +338,37 @@ def failing_run(command, **kwargs): text = err(capsys) assert "externally-managed-environment" in text assert "virtualenv" in text + + +def test_prune_failure_does_not_claim_space_was_freed( + fake_service, monkeypatch, capsys, tmp_path +): + from studiolink.models import PruneResult + + report = PruneReport( + dry_run=False, + imports=( + PruneResult(tmp_path / "model.gguf", 100, True, "permission denied", False), + ), + ) + monkeypatch.setattr(fake_service, "prune", lambda **kwargs: report) + assert cli.main(["prune"]) == 1 + output = out(capsys) + assert "Failed to remove" in output + assert "bytes freed" not in output + + +def test_prune_displays_destination_cleanup( + fake_service, monkeypatch, capsys, tmp_path +): + from studiolink.models import PruneResult + + report = PruneReport( + dry_run=False, + imports=(PruneResult(tmp_path / "model.gguf", 100, True, "obsolete", True),), + ) + monkeypatch.setattr(fake_service, "prune", lambda **kwargs: report) + assert cli.main(["prune"]) == 0 + output = out(capsys) + assert "model.gguf" in output + assert "100 bytes freed" in output diff --git a/tests/test_failure_paths.py b/tests/test_failure_paths.py index 7ed8915..cfdaeb3 100644 --- a/tests/test_failure_paths.py +++ b/tests/test_failure_paths.py @@ -16,6 +16,17 @@ from studiolink.state import StateStore +def test_prune_preserves_untracked_symlink_after_lost_state(library): + config, service, _, manifest, blob = library + service.sync(sync_all=True, link_mode=LinkMode.SYMBOLIC_LINK) + target = destination(config, service.scan()[0]) + config.state_file.unlink() + manifest.unlink() + blob.unlink() + service.prune() + assert target.is_file(), "prune broke an untracked LM Studio import" + + def test_state_write_failure_can_recover_existing_copy(library, monkeypatch): config, service, adapter, _, blob = library @@ -116,3 +127,77 @@ def test_real_cross_filesystem_import(make_config, fake_lmstudio, tmp_path, mode assert ( destination(config, service.scan()[0]).read_bytes() == blob.read_bytes() ) + + +def test_failed_prune_can_be_retried_without_losing_record(library, monkeypatch): + config, service, _, manifest, blob = library + service.sync(sync_all=True) + target = destination(config, service.scan()[0]) + alias = config.import_staging_dir / service.scan()[0].import_filename + manifest.unlink() + blob.unlink() + original = Path.unlink + + def denied(path, *args, **kwargs): + if path == target: + raise PermissionError("busy destination") + return original(path, *args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(Path, "unlink", denied) + report = service.prune() + assert not report.imports[0].removed + assert service.state.get_all_records() + assert alias.is_file() and target.is_file() + service.prune() + assert not service.state.get_all_records() + assert not target.exists() and not alias.exists() + + +def test_unavailable_lmstudio_directory_preserves_aliases_and_state(library): + config, service, _, manifest, blob = library + service.sync(sync_all=True) + alias = config.import_staging_dir / service.scan()[0].import_filename + manifest.unlink() + blob.unlink() + config.lmstudio_models_dir.rename(config.lmstudio_models_dir.with_name("unmounted")) + before = config.state_file.read_bytes() + with pytest.raises(RuntimeError, match="LM Studio models directory is unavailable"): + service.prune() + assert alias.is_file() + assert config.state_file.read_bytes() == before + + +def test_repull_preserves_untracked_symbolic_dependency(library): + from conftest import DIGEST_B + + config, service, _, _, _ = library + service.sync(sync_all=True, link_mode=LinkMode.SYMBOLIC_LINK) + alias = config.import_staging_dir / service.scan()[0].import_filename + manual = config.lmstudio_models_dir / "manual" / "model" / "saved.gguf" + manual.parent.mkdir(parents=True) + manual.symlink_to(alias) + original = manual.read_bytes() + write_manifest(config, digest=DIGEST_B) + write_blob(config, DIGEST_B) + assert ( + service.sync(sync_all=True, link_mode=LinkMode.SYMBOLIC_LINK)[0].status + == "synced" + ) + assert manual.read_bytes() == original + + +def test_prune_cleans_abandoned_copy_staging(library): + config, service, _, _, blob = library + model = service.scan()[0] + staging = config.ollama_models_dir / ".studiolink-imports" + staging.mkdir() + alias = staging / model.import_filename + os.link(blob, alias) + config.lmstudio_models_dir.mkdir(parents=True) + report = service.prune(dry_run=True) + assert alias.exists() + assert alias in {item.path for item in report.aliases} + service.prune() + assert not alias.exists() + assert blob.stat().st_nlink == 1 diff --git a/tests/test_import_lifecycle.py b/tests/test_import_lifecycle.py index 85d3b1f..aab7b72 100644 --- a/tests/test_import_lifecycle.py +++ b/tests/test_import_lifecycle.py @@ -1,10 +1,11 @@ """Exercise imports with real files, including the LM Studio destination.""" import os +import shutil from pathlib import Path import pytest -from conftest import DIGEST_A, write_blob, write_manifest +from conftest import DIGEST_A, DIGEST_B, write_blob, write_manifest from studiolink.models import ImportMode, LinkMode from studiolink.service import StudioLinkService @@ -42,6 +43,45 @@ def test_failed_import_does_not_pin_blob(library): assert blob.stat().st_nlink == 1 +def test_repull_does_not_leave_old_imports(library): + config, service, _, _, old_blob = library + service.sync(sync_all=True) + old_target = destination(config, service.scan()[0]) + old_alias = config.import_staging_dir / service.scan()[0].import_filename + write_manifest(config, digest=DIGEST_B) + write_blob(config, DIGEST_B) + assert service.sync(sync_all=True)[0].status == "synced" + assert not old_target.exists() + assert not old_alias.exists() + assert old_blob.stat().st_nlink == 1 + + +@pytest.mark.parametrize("mode", list(LinkMode)) +def test_prune_removes_managed_destination_and_state(library, mode): + config, service, _, manifest, blob = library + service.sync(sync_all=True, link_mode=mode) + target = destination(config, service.scan()[0]) + manifest.unlink() + blob.unlink() + service.prune() + assert not target.exists() and not target.is_symlink() + assert service.state.get_all_records() == {} + assert list(config.import_staging_dir.glob("*.gguf")) == [] + + +def test_unavailable_library_does_not_destroy_aliases(library): + config, service, _, _, _ = library + service.sync(sync_all=True, link_mode=LinkMode.SYMBOLIC_LINK) + model = service.scan()[0] + target = destination(config, model) + shutil.rmtree(config.ollama_models_dir) + try: + service.prune() + except (ValueError, OSError, RuntimeError): + pass + assert target.is_file() + + def test_sync_recovers_after_state_loss_without_reimport(library): config, service, adapter, _, _ = library service.sync(sync_all=True) @@ -110,6 +150,43 @@ def fail_link(*args, **kwargs): assert list(target.parent.glob(".studiolink-*")) == [] +def test_prune_does_not_delete_replaced_destination(library): + config, service, _, manifest, blob = library + service.sync(sync_all=True) + target = destination(config, service.scan()[0]) + target.unlink() + target.write_bytes(b"An unrelated replacement") + manifest.unlink() + blob.unlink() + service.prune() + assert target.read_bytes() == b"An unrelated replacement" + assert service.state.get_all_records() + + +def test_prune_dry_run_counts_shared_inode_once(library): + config, service, _, manifest, blob = library + service.sync(sync_all=True) + target = destination(config, service.scan()[0]) + expected_size = blob.stat().st_size + manifest.unlink() + blob.unlink() + report = service.prune(dry_run=True) + assert target.is_file() + assert ( + sum(p.size for p in (*report.aliases, *report.imports) if p.would_free) + == expected_size + ) + report = service.prune() + assert ( + sum( + p.size + for p in (*report.aliases, *report.imports) + if p.would_free and p.removed + ) + == expected_size + ) + + def test_copy_can_stage_across_filesystems_without_extra_copy(library, monkeypatch): import errno @@ -170,6 +247,43 @@ def test_sync_and_prune_do_not_overlap(library): assert service.sync(sync_all=True)[0].status == "synced" +def test_repull_keeps_alias_shared_by_other_model(library): + config, service, _, original_manifest, _ = library + original_manifest.unlink() + write_manifest(config, repository="a-b", tag="c") + write_manifest(config, repository="a", tag="b-c") + service.sync(sync_all=True, link_mode=LinkMode.SYMBOLIC_LINK) + other = next(m for m in service.scan() if m.canonical_name == "a:b-c") + target = destination(config, other) + write_manifest(config, repository="a-b", tag="c", digest=DIGEST_B) + write_blob(config, DIGEST_B) + assert ( + service.sync(model_names=["a-b:c"], link_mode=LinkMode.SYMBOLIC_LINK)[0].status + == "synced" + ) + assert target.is_file() + + +def test_failed_old_cleanup_can_be_retried_without_new_copy(library, monkeypatch): + config, service, adapter, _, _ = library + service.sync(sync_all=True, link_mode=LinkMode.COPY) + old_target = destination(config, service.scan()[0]) + write_manifest(config, digest=DIGEST_B) + write_blob(config, DIGEST_B) + real_unlink = Path.unlink + + def deny_old(path, *args, **kwargs): + if path == old_target: + raise PermissionError("busy") + return real_unlink(path, *args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(Path, "unlink", deny_old) + assert service.sync(sync_all=True, link_mode=LinkMode.COPY)[0].status == "error" + assert service.sync(sync_all=True, link_mode=LinkMode.COPY)[0].status == "synced" + assert len(adapter.calls) == 2 + + @pytest.mark.parametrize("mode", [LinkMode.HARD_LINK, LinkMode.COPY]) def test_sync_recovers_after_models_directory_is_moved(library, mode): from dataclasses import replace diff --git a/tests/test_import_transactions.py b/tests/test_import_transactions.py index c1eb6d7..aae4589 100644 --- a/tests/test_import_transactions.py +++ b/tests/test_import_transactions.py @@ -124,3 +124,23 @@ def unsupported(*args, **kwargs): publish_import(staged, target) assert target.read_bytes() == b"GGUFnew" assert not staged.exists() + + +def test_prune_reports_and_removes_abandoned_partial_import(library): + import json + + from studiolink.transactions import TRANSACTIONS + + config, service, _, _, _ = library + work = config.lmstudio_models_dir / TRANSACTIONS / "import-interrupted" + work.mkdir(parents=True) + target = work / "model.gguf" + target.write_bytes(b"GGUFpartial") + (work / "owner.json").write_text(json.dumps({"filename": target.name})) + report = service.prune(dry_run=True) + assert target.exists() + assert len(report.imports) == 1 + assert report.imports[0].would_free and not report.imports[0].removed + report = service.prune() + assert not target.exists() + assert report.imports[0].removed and report.imports[0].size == 11 diff --git a/tests/test_service.py b/tests/test_service.py index c631508..edfa342 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -19,6 +19,7 @@ def build_service(config, lmstudio): + config.lmstudio_models_dir.mkdir(parents=True, exist_ok=True) return StudioLinkService( config=config, ollama=OllamaAdapter(config), lmstudio=lmstudio ) @@ -285,6 +286,7 @@ def test_empty_but_available_library_removes_non_symbolic_records( ): config = make_config() config.ollama_manifests_dir.mkdir(parents=True) + config.ollama_blobs_dir.mkdir(parents=True) service = build_service(config, fake_lmstudio) store = StateStore(config.state_file) store.upsert(make_sync_record("removed:model", DIGEST_A)) From 2159be019e24e332abb5de2ba2bdade99c497663 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:30:37 +0530 Subject: [PATCH 06/11] fix: reuse identical model copies across tags - hard-link verified destination copies instead of copying weights again - reuse copies during link-mode conversion as well as initial imports - test inode sharing while keeping copies independent of Ollama blobs Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. --- src/studiolink/syncer.py | 37 +++++++++++++++++++++++++++++++++- tests/test_import_lifecycle.py | 24 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/studiolink/syncer.py b/src/studiolink/syncer.py index fda3075..60ef945 100644 --- a/src/studiolink/syncer.py +++ b/src/studiolink/syncer.py @@ -232,6 +232,15 @@ def _sync_one( stderr="", return_code=0, ) + elif link_mode is LinkMode.COPY and self._reuse_copy( + model, records, target + ): + import_result = ImportResult( + command=(), + stdout="Reused existing copy of identical weights", + stderr="", + return_code=0, + ) else: import_result = self.lmstudio.import_model( str(alias_path.absolute()), @@ -313,6 +322,32 @@ def _sync_one( record=record, ) + def _reuse_copy( + self, model: OllamaModel, records: dict[str, SyncRecord], target: Path + ) -> bool: + if model.blob_path is None: + return False + for record in records.values(): + if ( + record.digest != model.model_digest + or record.link_mode is not LinkMode.COPY + ): + continue + source = record_destination(record, self.config.lmstudio_models_dir) + if source == target or not healthy_destination( + source, model.blob_path, LinkMode.COPY, record + ): + continue + target.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, target) + except OSError as exc: + if exc.errno in (errno.EXDEV, errno.ENOTSUP): + continue + raise + return True + return False + def _replace_destination( self, source: Path, @@ -328,7 +363,7 @@ def _replace_destination( os.link(source.resolve(), temporary) elif mode is LinkMode.SYMBOLIC_LINK: temporary.symlink_to(source.absolute()) - else: + elif not self._reuse_copy(model, records, temporary): shutil.copyfile(source, temporary) os.replace(temporary, target) diff --git a/tests/test_import_lifecycle.py b/tests/test_import_lifecycle.py index aab7b72..a358184 100644 --- a/tests/test_import_lifecycle.py +++ b/tests/test_import_lifecycle.py @@ -115,6 +115,18 @@ def test_state_loss_does_not_adopt_unrelated_file(library): assert target.read_bytes() == b"other content" +def test_converting_identical_tags_to_copy_reuses_one_copy(library): + config, service, _, _, blob = library + write_manifest(config, tag="another") + service.sync(sync_all=True) + results = service.sync(sync_all=True, link_mode=LinkMode.COPY) + assert all(result.status == "synced" for result in results) + targets = [destination(config, model) for model in service.scan()] + assert len(targets) == 2 + assert targets[0].samefile(targets[1]) + assert not targets[0].samefile(blob) + + def test_direct_tags_sharing_a_target_can_change_to_copy(library): config, service, _, _, blob = library write_manifest(config, tag="another") @@ -224,6 +236,18 @@ def test_replaced_blob_same_digest_is_relinked(library): assert destination(config, service.scan()[0]).samefile(blob) +def test_copy_tags_share_one_destination_allocation(library): + config, service, _, _, blob = library + write_manifest(config, tag="latest") + assert all( + r.status == "synced" + for r in service.sync(sync_all=True, link_mode=LinkMode.COPY) + ) + targets = [destination(config, m) for m in service.scan()] + assert targets[0].samefile(targets[1]) + assert not targets[0].samefile(blob) + + def test_dry_run_does_not_replace_existing_alias(library): config, service, _, _, blob = library service.sync(sync_all=True) From 7bfd496c65335b10e94f5c4e5853653165b61e07 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:31:07 +0530 Subject: [PATCH 07/11] test: exercise real lms imports and mixed lifecycle operations - run 27 opt-in real-lms cases in isolated Linux mount namespaces - cover all link modes, state loss, mode changes, direct imports, and pruning - exercise 300 deterministic mixed lifecycle actions and final cleanup Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. Includes 27 sandboxed real-lms cases on Python 3.12. --- tests/test_lifecycle_sequences.py | 74 ++++++++++ tests/test_real_lms.py | 215 ++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+) create mode 100644 tests/test_lifecycle_sequences.py create mode 100644 tests/test_real_lms.py diff --git a/tests/test_lifecycle_sequences.py b/tests/test_lifecycle_sequences.py new file mode 100644 index 0000000..655bdf9 --- /dev/null +++ b/tests/test_lifecycle_sequences.py @@ -0,0 +1,74 @@ +"""Repeat mixed user actions to catch interactions between lifecycle operations.""" + +import hashlib +import random + +import pytest +from conftest import write_blob, write_manifest + +from studiolink.models import LinkMode + + +@pytest.mark.parametrize("seed", range(10)) +def test_mixed_operations_remain_repairable_and_reclaimable(library, seed): + config, service, _, original_manifest, _ = library + original_manifest.unlink() + rng = random.Random(seed) + versions = {} + for tag in ("small", "latest"): + content = b"GGUF" + bytes([1]) * 8 + digest = "sha256:" + hashlib.sha256(content).hexdigest() + write_blob(config, digest).write_bytes(content) + write_manifest(config, tag=tag, digest=digest) + versions[tag] = 1 + for step in range(30): + action = rng.choice( + ("repull", "remove-target", "state-loss", "dry-run", "sync") + ) + if action == "repull": + tag = rng.choice(list(versions)) + versions[tag] += 1 + content = b"GGUF" + bytes([versions[tag]]) * 8 + digest = "sha256:" + hashlib.sha256(content).hexdigest() + # Ollama reuses an existing content-addressed blob. Do not rewrite + # its inode (and every hard-linked import) when another tag uses it. + blob = config.ollama_blobs_dir / digest.replace(":", "-") + if not blob.exists(): + write_blob(config, digest).write_bytes(content) + write_manifest(config, tag=tag, digest=digest) + elif action == "remove-target": + targets = list(config.lmstudio_models_dir.glob("*/*/*.gguf")) + if targets: + rng.choice(targets).unlink() + elif action == "state-loss": + # Losing state after an update leaves obsolete imports untracked: + # clean them while ownership is still available before this step. + service.prune() if config.lmstudio_models_dir.exists() else None + config.state_file.unlink(missing_ok=True) + mode = rng.choice(list(LinkMode)) + if action == "dry-run": + before = { + p: p.lstat().st_ino + for p in config.lmstudio_models_dir.glob("*/*/*.gguf") + } + service.sync(sync_all=True, link_mode=mode, dry_run=True) + after = { + p: p.lstat().st_ino + for p in config.lmstudio_models_dir.glob("*/*/*.gguf") + } + assert before == after + results = service.sync(sync_all=True, link_mode=mode) + assert all(item.status in ("synced", "skipped") for item in results), ( + step, + action, + mode, + [(item.status, item.message) for item in results], + ) + assert all(entry.synced for entry in service.status()) + # All library models removed: every tracked destination must disappear. + for model in service.scan(): + model.manifest_path.unlink() + service.prune() + assert not service.state.get_all_records() + assert not list(config.lmstudio_models_dir.glob("*/*/*.gguf")) + assert not list(config.import_staging_dir.glob("*.gguf")) diff --git a/tests/test_real_lms.py b/tests/test_real_lms.py new file mode 100644 index 0000000..d305995 --- /dev/null +++ b/tests/test_real_lms.py @@ -0,0 +1,215 @@ +"""Opt-in live CLI checks, isolated from the user's LM Studio by mount namespaces. + +Run with STUDIOLINK_TEST_REAL_LMS=1. Linux, bubblewrap and lms are required. +No model weights are loaded; the tiny GGUF fixtures exercise importing only. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import struct +import subprocess +import sys +from pathlib import Path + +import pytest + +from studiolink.models import LinkMode + + +@pytest.fixture +def real_cli(tmp_path): + if os.environ.get("STUDIOLINK_TEST_REAL_LMS") != "1": + pytest.skip("set STUDIOLINK_TEST_REAL_LMS=1 for isolated real-lms tests") + bwrap = shutil.which("bwrap") + lms = shutil.which("lms") + if not sys.platform.startswith("linux") or not bwrap or not lms: + pytest.skip("real-lms sandbox requires Linux, bubblewrap and lms") + probe = subprocess.run( + [bwrap, "--ro-bind", "/", "/", "--unshare-net", "--", "true"], + capture_output=True, + text=True, + ) + if probe.returncode: + pytest.skip(f"bubblewrap unavailable: {probe.stderr.strip()}") + root = tmp_path / "library with spaces" + root.mkdir() + lmhome = root / "lmstudio-home" + lmhome.mkdir() + models = root / "lmstudio-models" + (lmhome / "settings.json").write_text(json.dumps({"downloadsFolder": str(models)})) + binary = root / "real-lms" + binary.touch() + source = root / "ollama" + state = root / "state" + env = os.environ.copy() + env.update( + { + "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src"), + "STUDIOLINK_OLLAMA_MODELS_DIR": str(source), + "STUDIOLINK_LMSTUDIO_MODELS_DIR": str(models), + "STUDIOLINK_STATE_DIR": str(state), + "STUDIOLINK_LMS_EXE": str(binary), + } + ) + command = [ + bwrap, + "--die-with-parent", + "--unshare-net", + "--ro-bind", + "/", + "/", + "--dev", + "/dev", + "--tmpfs", + "/tmp", + "--bind", + str(root), + str(root), + "--ro-bind", + lms, + str(binary), + "--bind", + str(lmhome), + str(Path.home() / ".lmstudio"), + "--", + sys.executable, + "-m", + "studiolink", + ] + + class CLI: + def run(self, *args, expected=0): + completed = subprocess.run( + [*command, *args, "--json"], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == expected, ( + completed.stdout[:2200], + completed.stderr[:2200], + ) + return json.loads(completed.stdout) + + def model(self, *, tag="small", payload=b""): + content = struct.pack("<4sIQQ", b"GGUF", 3, 0, 0) + payload + digest = "sha256:" + hashlib.sha256(content).hexdigest() + blob = source / "blobs" / digest.replace(":", "-") + blob.parent.mkdir(parents=True, exist_ok=True) + blob.write_bytes(content) + manifest = ( + source + / "manifests" + / "registry.ollama.ai" + / "library" + / "fixture" + / tag + ) + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text( + json.dumps( + { + "layers": [ + { + "mediaType": "application/vnd.ollama.image.model", + "digest": digest, + "size": len(content), + } + ] + } + ) + ) + return blob, manifest + + def target(self): + paths = list(models.rglob("*.gguf")) + assert len(paths) == 1, paths + return paths[0] + + cli = CLI() + cli.root, cli.models, cli.source, cli.state = root, models, source, state + return cli + + +@pytest.mark.parametrize("mode", list(LinkMode)) +def test_real_lms_full_lifecycle(real_cli, mode): + blob, manifest = real_cli.model() + assert ( + real_cli.run("sync", "--all", "--dry-run", mode.lms_flag)[0]["status"] + == "dry-run" + ) + assert not list(real_cli.models.rglob("*.gguf")) + assert real_cli.run("sync", "--all", mode.lms_flag)[0]["status"] == "synced" + target = real_cli.target() + assert target.read_bytes() == blob.read_bytes() + assert target.is_symlink() == (mode is LinkMode.SYMBOLIC_LINK) + assert target.samefile(blob) == (mode is not LinkMode.COPY) + assert real_cli.run("sync", "--all", mode.lms_flag)[0]["status"] == "skipped" + target.unlink() + assert not real_cli.run("status")[0]["synced"] + assert real_cli.run("sync", "--all", mode.lms_flag)[0]["status"] == "synced" + (real_cli.state / "state.json").unlink() + assert real_cli.run("sync", "--all", mode.lms_flag)[0]["status"] == "synced" + new_blob, manifest = real_cli.model(payload=b"updated weights") + assert real_cli.run("sync", "--all", mode.lms_flag)[0]["status"] == "synced" + assert not target.exists() and not target.is_symlink() + target = real_cli.target() + assert target.read_bytes() == new_blob.read_bytes() + manifest.unlink() + new_blob.unlink() + real_cli.run("prune", "--dry-run") + assert target.is_file() + real_cli.run("prune") + assert not list(real_cli.models.rglob("*.gguf")) + assert not list((real_cli.state / "imports").glob("*.gguf")) + assert json.loads((real_cli.state / "state.json").read_text())["sync_records"] == {} + + +@pytest.mark.parametrize("first", list(LinkMode)) +@pytest.mark.parametrize("second", list(LinkMode)) +@pytest.mark.parametrize("lose_state", [False, True]) +def test_real_lms_mode_conversion(real_cli, first, second, lose_state): + blob, _ = real_cli.model() + real_cli.run("sync", "--all", first.lms_flag) + if lose_state: + (real_cli.state / "state.json").unlink() + real_cli.run("sync", "--all", second.lms_flag) + target = real_cli.target() + assert target.read_bytes() == blob.read_bytes() + assert target.is_symlink() == (second is LinkMode.SYMBOLIC_LINK) + assert target.samefile(blob) == (second is not LinkMode.COPY) + + +@pytest.mark.parametrize("initial", [None, LinkMode.HARD_LINK, LinkMode.SYMBOLIC_LINK]) +def test_real_lms_identical_tags_use_one_copy(real_cli, initial): + real_cli.model(tag="one") + real_cli.model(tag="two") + if initial is not None: + real_cli.run("sync", "--all", initial.lms_flag) + assert all( + row["status"] == "synced" for row in real_cli.run("sync", "--all", "--copy") + ) + targets = list(real_cli.models.rglob("*.gguf")) + assert len(targets) == 2 + assert targets[0].samefile(targets[1]) + + +@pytest.mark.parametrize("mode", list(LinkMode)) +def test_real_lms_direct_import(real_cli, mode): + blob, manifest = real_cli.model() + results = real_cli.run("sync", "--all", "--direct", mode.lms_flag) + assert results[0]["status"] == "synced" + target = real_cli.models / "ollama/fixture" / blob.name + assert target.read_bytes() == blob.read_bytes() + assert target.is_symlink() == (mode is LinkMode.SYMBOLIC_LINK) + assert target.samefile(blob) == (mode is not LinkMode.COPY) + assert not (real_cli.state / "imports").exists() + manifest.unlink() + real_cli.run("prune") + assert not target.exists() and not target.is_symlink() + assert blob.is_file() From 8944a210a1394b2995ea43b17bfa4cfbdae73227 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:31:37 +0530 Subject: [PATCH 08/11] docs: explain import recovery and storage cleanup - document verified imports, transactional recovery, and copy deduplication - explain managed cleanup, cross-volume staging, and dependency protection - record per-commit validation and remaining native-platform limits Validation: pytest passes on Python 3.12/3.13/3.14; Ruff, formatting, Pyright, and distribution builds pass. Includes 27 sandboxed real-lms cases on Python 3.12. --- README.md | 33 +++---- docs/import-lifecycle-validation.md | 130 ++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 docs/import-lifecycle-validation.md diff --git a/README.md b/README.md index 6375dba..2c6805b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ StudioLink is a Python CLI tool that bridges Ollama and LM Studio. It treats Oll - 🔄 **Sync** - Import models into LM Studio with one command - 📊 **Status** - Track which models are synced and their current state - 🩺 **Doctor** - Verify prerequisites and diagnose issues -- 🧹 **Prune** - Safely remove orphaned aliases without breaking symbolic imports +- 🧹 **Prune** - Remove obsolete managed imports and aliases to release pinned disk space - 🔗 **Smart linking** - Uses hard links by default; pass `--copy` when volumes differ - 💾 **State tracking** - Remembers what's been synced to avoid re-importing @@ -18,7 +18,7 @@ StudioLink is a Python CLI tool that bridges Ollama and LM Studio. It treats Oll Before using StudioLink, ensure you have: -1. **Ollama** installed with a local model library (StudioLink detects current platform defaults and honors `OLLAMA_MODELS`) +1. **Ollama** installed with a local model library (StudioLink detects platform defaults and honors `OLLAMA_MODELS`) 2. **LM Studio** with the `lms` CLI installed 3. **Python 3.12+** @@ -135,9 +135,9 @@ Discovered 3 model(s); 1 tracked as synced. ``` ### `sdl prune` -Remove aliases and sync state that are no longer needed, after first verifying that the Ollama library was scanned completely. +Remove tracked LM Studio imports, staging aliases, and sync state for models that Ollama no longer has or has replaced. -Space recovery depends on the import mode. A copied import can leave the staging alias as the last reference to Ollama's former blob, while an LM Studio hard link continues to retain the model data after the staging alias is removed. Aliases required by symbolic-link imports are preserved so LM Studio is not left with a broken link. +Hard-linked weights remain on disk until **all** links are removed, including the LM Studio destination. `sdl prune` removes both the tracked destination and its staging alias. It preserves files whose identity has changed and refuses cleanup after an incomplete source scan or when either model library is unavailable. It also recovers abandoned import transactions and temporary copy aliases. Files already present in LM Studio without a verifiable StudioLink record are left alone. ```powershell # Preview what would be removed @@ -214,16 +214,17 @@ Default paths are platform-aware; every one can be overridden with an environmen | LM Studio models | LM Studio's configured `downloadsFolder` | LM Studio's configured `downloadsFolder` | LM Studio's configured `downloadsFolder` | `STUDIOLINK_LMSTUDIO_MODELS_DIR` | | State directory | `~\.studiolink` | `~/.studiolink` | `~/.studiolink` | `STUDIOLINK_STATE_DIR` | +The LM Studio models override must match the models directory selected in LM Studio; it does not change where `lms import` writes files. A mismatch is reported before importing. + ## How It Works 1. **Scanning** - Reads manifest files from the configured Ollama models directory 2. **Blob Resolution** - Locates GGUF blobs in Ollama's content-addressed blob directory -3. **Validation** - Verifies blobs start with GGUF magic bytes -4. **Import Aliases** - Creates digest-keyed, human-readable `.gguf` aliases. Copy mode uses a temporary alias on Ollama's filesystem so the state and LM Studio directories may live on other volumes -5. **LM Studio Import** - Uses `lms import` CLI to import models -6. **State Tracking** - Saves sync records and the expected LM Studio target to `~/.studiolink/state.json` (written atomically) -7. **Reconciliation** - Status and sync verify that the expected LM Studio target still exists instead of trusting state alone -8. **Pruning** - Pruning is blocked after unavailable or incomplete Ollama scans and preserves aliases required by symbolic imports +3. **Validation** - Checks SHA-256 digest syntax, GGUF magic bytes, and the blob size declared in the manifest +4. **Import Aliases** - Creates digest-keyed, human-readable `.gguf` aliases. Copy mode uses a temporary alias on Ollama's filesystem, removed after the import; other modes use `~/.studiolink/imports/` +5. **LM Studio Import** - Uses `lms import` to stage and verify the requested link/copy, then publishes the destination without overwriting an existing file. Failed imports clean up partial files; subsequent sync or prune recovers abandoned transactions after a process crash +6. **State Tracking** - Saves destination paths and file identities to `~/.studiolink/state.json` (written atomically), verifies imports before skipping, and repairs missing destinations. Sync and prune operations share a process lock. +7. **Cleanup** - Successful replacement imports remove the previous managed destination and alias. `sdl prune` removes obsolete tracked imports and orphaned aliases; its dry run counts shared file data once. ## Troubleshooting @@ -234,16 +235,16 @@ ollama pull ``` ### Hard link fails (cross-volume) -Hard-link imports require the Ollama blob store, StudioLink alias directory, and LM Studio models directory to live on the same filesystem/volume. If they do not, retry with copy mode; copy mode creates its temporary naming alias on Ollama's filesystem and can copy to LM Studio on another volume: +Hard links require the Ollama blob store, the StudioLink state directory, and the LM Studio models directory to live on the same filesystem/volume. If they don't, the sync reports a hard-link error; retry with copy mode instead: ```powershell sdl sync --copy ``` -The StudioLink state directory does not need to share Ollama's volume when using copy mode. +Copy mode creates its temporary naming alias on Ollama's filesystem, so the state directory and LM Studio library may live on other volumes without requiring symbolic-link permission. Symbolic-link mode uses a symlink for cross-volume staging and requires permission to create symlinks on Windows. ### Deleted a model in Ollama but disk space wasn't freed -Run `sdl prune` to remove aliases that are no longer needed. If LM Studio imported the model with `--hard-link`, its final hard link intentionally continues to retain the data; removing only the staging alias cannot free those bytes. Symbolic-link aliases are preserved while LM Studio depends on them. +Both LM Studio imports and StudioLink staging aliases can keep a deleted Ollama blob alive. Run `sdl prune --dry-run` to preview cleanup, then `sdl prune` to remove the obsolete managed files. Untracked LM Studio files may still hold links and need separate inspection. ### LM Studio import fails Run `sdl doctor` to verify: @@ -256,8 +257,10 @@ Run `sdl doctor` to verify: - **Ollama as source of truth** - StudioLink never changes Ollama manifests or content-addressed blobs; copy mode may briefly create a `.studiolink-imports` alias directory on the same filesystem - **Hard links by default** - Space-efficient, creates references rather than copies - **No silent fallbacks** - If a hard link is impossible (cross-volume), the sync fails loudly; use `--copy` explicitly -- **State persistence** - Tracks imports to avoid redundant operations; state is written atomically and corrupt records are skipped, never crash -- **Safe pruning** - Orphaned aliases are cleaned up only after a complete source scan; symbolic-link dependencies are retained +- **State persistence** - Tracks and verifies actual imports to avoid redundant operations; state is written atomically and corrupt records are skipped. Verified existing files can restore lost state. +- **Copy deduplication** - Tags with identical weights share one destination copy through hard links when the destination filesystem supports them. +- **Dry runs** - Preview sync actions without replacing aliases or invoking an import. +- **Prune over leak** - Obsolete tracked destinations and orphaned aliases are cleaned up with `sdl prune` instead of pinning disk space forever - **CLI-only** - No GUI, designed for automation and scripting ## License diff --git a/docs/import-lifecycle-validation.md b/docs/import-lifecycle-validation.md new file mode 100644 index 0000000..eec9dd3 --- /dev/null +++ b/docs/import-lifecycle-validation.md @@ -0,0 +1,130 @@ +# Import and storage validation — 2026-09-30 + +Validation was completed before committing on `fix/import-lifecycle-validation`, +based on `f9df9a3`. Local `main` remained at `221229d` throughout validation. + +## Focused commit validation + +The original combined commit (`ecb7fba`) is preserved locally at +`backup/import-lifecycle-before-split`. It was replaced on the fix branch with +focused commits. Every implementation commit includes its regression tests; +the final test commit adds the real-CLI and mixed-operation suites. + +Before each commit, the complete suite present at that point passed on all three +Python versions, along with Ruff lint/format checks, Pyright, a source distribution +and wheel build, and `git diff --check`. + +| Commit | Scope | Python 3.12 | Python 3.13 | Python 3.14 | +| --- | --- | --- | --- | --- | +| `64dbaa9` | validate Ollama blobs and surface incomplete scans | 139 passed | 139 passed | 139 passed | +| `b4a193d` | serialize sync and prune across processes | 142 passed | 142 passed | 142 passed | +| `966d460` | recover interrupted LM Studio imports | 155 passed | 155 passed | 155 passed | +| `d7e5f67` | verify and repair imported model files | 182 passed | 182 passed | 182 passed | +| `fb4df6f` | reclaim obsolete imports without breaking dependent links | 199 passed | 199 passed | 199 passed | +| `83164b7` | reuse identical model copies across tags | 201 passed | 201 passed | 201 passed | +| `f85820e` | exercise real lms imports and mixed lifecycle operations | 238 passed | 211 passed, 27 skipped | 211 passed, 27 skipped | + +The documentation commit repeats the final checks. The rebuilt `src/` and +`tests/` trees match the original combined commit exactly; documentation adds +this validation record. Native-platform limits below remain unchanged. + +## Results + +| Check | Result | +| --- | --- | +| Linux, Python 3.12.14, including real `lms` | 238 passed | +| Linux, Python 3.13.15 | 211 passed, 27 opt-in real-CLI tests skipped | +| Linux, Python 3.14.7 | 211 passed, 27 opt-in real-CLI tests skipped | +| Ruff lint and formatting | Passed | +| Pyright | 0 errors, 0 warnings | +| Source distribution and wheel build | Passed | +| Wheel installed in a fresh Python 3.12 environment outside the checkout | Passed | +| Installed `sdl` version/help, scan, status, dry-run sync | Passed | +| Git whitespace/conflict checks | Passed | + +The live CLI was `lms` commit `69d945a`. Its tests run in a Linux bubblewrap +mount namespace with the host filesystem read-only, a temporary writable fixture +library, an isolated LM Studio settings directory, and networking disabled. +The user's model libraries are not modified. The 27 cases cover the three link +modes, destination repair, lost state, repulls, pruning, all nine mode transitions +with and without state loss, identical-tag copy deduplication, and direct imports. + +## Failure cases exercised + +| Scenario | Method and assertion | +| --- | --- | +| Partial import, disk-full error, timeout, interruption | Injected subprocess failures after writing a real partial file; partial data and staging aliases disappear, retry succeeds | +| Process dies during a transaction | A child writes a partial import and calls `os._exit`; recovery removes its abandoned files | +| Concurrent operations and dead process locks | A real child holds the OS lock; another operation is refused; killing the child releases the lock | +| Full disk during state persistence | Injected `ENOSPC`; previous JSON survives atomic replacement failure; an existing completed copy is adopted without another import | +| Cross-filesystem imports | Real paths on separate filesystems (`tmp_path` and `/dev/shm`); hard-link mode fails without silently copying, copy and symbolic-link modes work | +| Unsupported destination hard links | Injected unsupported-link error exercises the actual Linux no-overwrite rename fallback | +| Missing or changed destination, source replacement, moved library | Filesystem tests verify status detects the change and sync repairs or safely refuses it | +| Unavailable library or incomplete source scan | Pruning refuses to delete imports, aliases, or state | +| Foreign replacement or untracked symbolic dependency | User files and aliases required by other imports remain intact | +| Prune permission failure | The failed item retains its record; a subsequent prune can finish | +| Identical tags, including mode conversion | Destination inode identity proves one independent copy is shared instead of duplicating weights | +| Shared direct-import destination | Both tags remain healthy after switching the shared target to copy mode | +| Removed or superseded models | Managed destination and alias are removed; shared-inode accounting avoids counting the same data twice | +| Dry run | Existing imported files and aliases are not replaced and no import is invoked | +| Mixed lifecycle operations | Ten deterministic seeds, 30 actions each: repulls, target deletion, state loss, dry runs, and mode changes, followed by complete tracked cleanup | + +Focused regression tests were observed failing before the fixes for lost-state +mode conversion, duplicate copies during conversion, and shared direct targets. +The mixed-operation tests also exposed an unrealistic fixture rewrite of an +existing content-addressed blob; the fixture now reuses that blob as Ollama does. + +## Behavior changed + +- Sync and status verify the actual destination, requested link mode, and source + relationship instead of trusting a saved digest alone. +- Imports stage in a recoverable transaction and publish only after verification. + Failed imports and abandoned transactions no longer leave permanent partial + models. Mode conversion replaces the managed file atomically. +- Repulls and pruning remove verified obsolete destinations as well as aliases, + so an overlooked hard link cannot indefinitely pin obsolete weights. +- Copy imports reuse verified identical weights across tags, including when + converting existing links to copies. Temporary copy aliases are cleaned up. +- OS locks serialize sync/prune operations sharing a state file. Lost state can + be recovered from a verified existing destination without another full copy. +- Malformed manifests, incomplete scans, missing mounts, mismatched LM Studio + paths, and replaced files receive more defensive handling. + +## Reproduction + +From the checkout, with `uv` available: + +```bash +STUDIOLINK_TEST_REAL_LMS=1 uv run --no-project --python 3.12 --with pytest python -m pytest -q +uv run --no-project --python 3.13 --with pytest python -m pytest -q +uv run --no-project --python 3.14 --with pytest python -m pytest -q +uv run --no-project --python 3.12 --with ruff ruff check src tests +uv run --no-project --python 3.12 --with ruff ruff format --check src tests +uv run --no-project --python 3.12 --with pyright pyright +uv run --no-project --python 3.12 --with build python -m build +git diff --check +``` + +The real-CLI suite requires Linux, working bubblewrap user/mount namespaces, and +`lms` on PATH. It explicitly skips when prerequisites or its opt-in flag are absent; +check the skip count before treating a run as live-CLI validation. Package checks +built both distributions, installed the wheel into a temporary virtual environment, +verified imports came from site-packages, and ran the installed `sdl` entry point +outside the checkout. Temporary distributions and environments were removed. + +## Remaining validation limits + +- Native Windows and macOS were not available. Windows locking, symlink privileges, + and platform-specific rename behavior still need native runs. Existing CI defines + Linux/Windows Python 3.12–3.14 jobs; those remote jobs have not been run for this + changes. +- Fixtures contain tiny GGUF headers and payloads. These checks validate importing + and file lifecycle, not model loading/inference or multi-gigabyte throughput. +- Disk exhaustion, timeouts, and permission failures are injected. The tests do not + fill a physical SSD, cut power, or kill the real CLI during a large copy. +- Actual exFAT, NTFS, network shares, and filesystem durability after power loss + were not tested. Inode/link-count assertions establish sharing and logical + reclamation, not a hardware SSD space measurement. +- Files with lost ownership evidence are deliberately retained when their content + or dependencies cannot be verified; arbitrary pre-existing duplicates are not + automatically deleted. From a99b2e46f40b248dbf029b5017accf5310f4e350 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:32:37 +0530 Subject: [PATCH 09/11] fix: scan Ollama 0.40 manifests-v2 alongside legacy manifests Ollama 0.40 writes named manifests to manifests-v2//// as links to content-addressed manifest blobs; 0.40.16 `ollama create` and `ollama cp` no longer write the legacy tree at all. StudioLink only scanned manifests/, so those models were invisible and `sdl prune` treated their LM Studio imports as obsolete and deleted them. - Scan both trees, preferring v2 and falling back to legacy only when the v2 name is absent or a dangling link; map ollama.com to registry.ollama.ai so canonical names and sync records stay stable, and decode %3A-encoded registry ports. - Verify v2 links and referenced child manifests against their digests. - Select the llamacpp GGUF child of a manifest list, then ggml. - Skip the legacy llamacpp: downgrade shadow tags. - Report MLX/safetensors-only models as invalid with a clear message, keeping them in the scan so prune never removes their records. - Doctor accepts either manifests directory. Co-Authored-By: Claude --- README.md | 7 +- src/studiolink/cli.py | 6 +- src/studiolink/config.py | 10 + src/studiolink/ollama_adapter.py | 359 +++++++++++++++++++++++----- src/studiolink/service.py | 10 +- tests/test_config.py | 1 + tests/test_ollama_adapter.py | 8 +- tests/test_ollama_manifests_v2.py | 373 ++++++++++++++++++++++++++++++ tests/test_service.py | 8 +- 9 files changed, 707 insertions(+), 75 deletions(-) create mode 100644 tests/test_ollama_manifests_v2.py diff --git a/README.md b/README.md index c26b69c..c2586a1 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ sdl doctor --json **Checks include:** - Ollama executable exists - LM Studio CLI (`lms`) exists -- Ollama manifests and blobs directories exist +- Ollama manifests (`manifests-v2/` from Ollama 0.40+, or legacy `manifests/`) and blobs directories exist - LM Studio models directory exists - Hard-link filesystem/volume compatibility - Hard-link permission on blobs (a real link is created and removed again, catching restrictions such as Linux `fs.protected_hardlinks`) @@ -220,7 +220,7 @@ The LM Studio models override must match the models directory selected in LM Stu ## How It Works -1. **Scanning** - Reads manifest files from the configured Ollama models directory +1. **Scanning** - Reads named manifests from both `manifests-v2/` (Ollama 0.40+, links to content-addressed manifest blobs) and the legacy `manifests/` tree, preferring v2 when a name exists in both. For Ollama manifest lists it picks the llama.cpp GGUF child, falling back to the `ggml` child; internal `llamacpp:` downgrade tags are ignored 2. **Artifact Resolution** - Locates the primary model and optional projector GGUF in Ollama's content-addressed blob directory 3. **Validation** - Checks every required artifact's SHA-256 digest syntax, GGUF magic bytes, and the blob size declared in the manifest 4. **Import Aliases** - Creates digest-keyed, human-readable `.gguf` aliases; projectors use the LM Studio-recognized `mmproj-` prefix. Copy mode uses temporary aliases on Ollama's filesystem, removed after the import; other modes use `~/.studiolink/imports/` @@ -237,6 +237,9 @@ The model manifest exists but the actual blob file has been deleted or purged by ollama pull ``` +### "Ollama MLX (safetensors) models are not supported" +The model only has an MLX build in Ollama (common on Apple Silicon), which has no GGUF file for LM Studio to import. Pull a GGUF variant of the model, or download it in LM Studio directly. + ### Hard link fails (cross-volume) Hard links require the Ollama blob store, the StudioLink state directory, and the LM Studio models directory to live on the same filesystem/volume. If they don't, the sync reports a hard-link error; retry with copy mode instead: diff --git a/src/studiolink/cli.py b/src/studiolink/cli.py index d0a3913..442a1b0 100644 --- a/src/studiolink/cli.py +++ b/src/studiolink/cli.py @@ -154,7 +154,11 @@ def main(argv: list[str] | None = None) -> int: service = StudioLinkService() logger.debug("Configuration loaded:") - logger.debug(" Ollama manifests: %s", service.config.ollama_manifests_dir) + logger.debug( + " Ollama manifests: %s (v2), %s (legacy)", + service.config.ollama_manifests_v2_path, + service.config.ollama_manifests_dir, + ) logger.debug(" Ollama blobs: %s", service.config.ollama_blobs_dir) logger.debug(" LM Studio models: %s", service.config.lmstudio_models_dir) logger.debug(" State file: %s", service.config.state_file) diff --git a/src/studiolink/config.py b/src/studiolink/config.py index 4e658cf..7988e50 100644 --- a/src/studiolink/config.py +++ b/src/studiolink/config.py @@ -63,6 +63,15 @@ class StudioLinkConfig: state_file: Path import_staging_dir: Path default_link_mode: LinkMode = LinkMode.HARD_LINK + # Ollama 0.40+ writes named manifests to manifests-v2/; None derives it + # from ollama_models_dir. + ollama_manifests_v2_dir: Path | None = None + + @property + def ollama_manifests_v2_path(self) -> Path: + if self.ollama_manifests_v2_dir is not None: + return self.ollama_manifests_v2_dir + return self.ollama_models_dir / "manifests-v2" @classmethod def from_env(cls) -> "StudioLinkConfig": @@ -85,6 +94,7 @@ def from_env(cls) -> "StudioLinkConfig": ), ollama_models_dir=ollama_models_dir, ollama_manifests_dir=ollama_models_dir / "manifests", + ollama_manifests_v2_dir=ollama_models_dir / "manifests-v2", ollama_blobs_dir=ollama_models_dir / "blobs", lmstudio_models_dir=_expand_path( "STUDIOLINK_LMSTUDIO_MODELS_DIR", diff --git a/src/studiolink/ollama_adapter.py b/src/studiolink/ollama_adapter.py index 9075d49..0933d20 100644 --- a/src/studiolink/ollama_adapter.py +++ b/src/studiolink/ollama_adapter.py @@ -1,9 +1,11 @@ from __future__ import annotations +import hashlib import json import logging import os import re +from dataclasses import dataclass from pathlib import Path from studiolink.config import StudioLinkConfig @@ -21,8 +23,63 @@ "application/vnd.ollama.image.draft": "draft", "application/vnd.ollama.image.tensor": "tensor", } +MANIFEST_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v2+json" +MANIFEST_LIST_MEDIA_TYPE = "application/vnd.ollama.manifest.list.v2+json" +# Ollama 0.40+ manifest lists hold one child per runner. llamacpp children are +# the clean llama.cpp-compatible GGUF that LM Studio can load; ggml children are +# Ollama's original GGUF layout, which older StudioLink releases imported. +GGUF_RUNNER_PREFERENCE = ("llamacpp", "ggml") +MLX_RUNNER = "mlx" +SAFETENSORS_FORMAT = "safetensors" +MLX_ONLY_ISSUE = ( + "Ollama MLX (safetensors) models are not supported; LM Studio imports need " + "a GGUF build" +) GGUF_MAGIC = b"GGUF" SHA256_DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-fA-F]{64}$") +BLOB_FILENAME_PATTERN = re.compile(r"^sha256-([0-9a-fA-F]{64})$") +HEX_DIGEST_PATTERN = re.compile(r"^[0-9a-fA-F]{64}$") +DEFAULT_REGISTRY = "registry.ollama.ai" +# manifests-v2 stores the default public registry under its canonical host. +PUBLIC_REGISTRY_HOSTS = frozenset({DEFAULT_REGISTRY, "ollama.com"}) +LEGACY_LAYOUT = "legacy" +V2_LAYOUT = "v2" + + +@dataclass(slots=True, frozen=True) +class _ManifestEntry: + registry: str + namespace: str + repository: str + tag: str + path: Path + layout: str + + @property + def key(self) -> tuple[str, str, str, str]: + # Ollama compares names case-insensitively. + return ( + self.registry.lower(), + self.namespace.lower(), + self.repository.lower(), + self.tag.lower(), + ) + + @property + def is_converted_shadow(self) -> bool: + # Ollama's compat migration writes a legacy "llamacpp:" tag + # per converted child as a downgrade anchor; it is not a user model. + return ( + self.registry == DEFAULT_REGISTRY + and self.namespace == "library" + and self.repository == "llamacpp" + and HEX_DIGEST_PATTERN.fullmatch(self.tag) is not None + ) + + +class _ManifestBlobMissing(Exception): + pass + logger = logging.getLogger("studiolink") @@ -35,10 +92,15 @@ def scan_models(self) -> list[OllamaModel]: return list(self.scan_report().models) def scan_report(self) -> OllamaScanReport: - manifests_dir = self.config.ollama_manifests_dir - logger.debug("Scanning manifests directory: %s", manifests_dir) - if not manifests_dir.is_dir(): - message = f"Manifests directory does not exist: {manifests_dir}" + roots = ( + (V2_LAYOUT, self.config.ollama_manifests_v2_path), + (LEGACY_LAYOUT, self.config.ollama_manifests_dir), + ) + present = [(layout, root) for layout, root in roots if root.is_dir()] + if not present: + message = "Manifests directory does not exist: " + " or ".join( + str(root) for _, root in roots + ) logger.warning(message) return OllamaScanReport( source_available=False, @@ -46,45 +108,65 @@ def scan_report(self) -> OllamaScanReport: errors=(message,), ) - models: list[OllamaModel] = [] errors: list[str] = [] - - def scan_error(exc: OSError) -> None: - raise exc - - try: - manifest_files = sorted( - Path(directory) / name - for directory, _, names in os.walk(manifests_dir, onerror=scan_error) - for name in names - ) - except OSError as exc: - message = f"Could not enumerate Ollama manifests: {exc}" - logger.warning(message) + entries: dict[str, dict[tuple[str, str, str, str], _ManifestEntry]] = {} + enumerated = 0 + for layout, root in present: + logger.debug("Scanning %s manifests directory: %s", layout, root) + try: + paths = self._enumerate(root) + except OSError as exc: + message = f"Could not enumerate Ollama manifests in {root}: {exc}" + logger.warning(message) + errors.append(message) + continue + enumerated += 1 + logger.debug("Found %d %s manifest file(s)", len(paths), layout) + found = entries.setdefault(layout, {}) + for path in paths: + entry = self._entry_for_path(root, path, layout) + if entry is None: + logger.debug("Skipped manifest: %s", path) + elif entry.is_converted_shadow: + logger.debug("Skipped converted shadow manifest: %s", path) + else: + found.setdefault(entry.key, entry) + if not enumerated: return OllamaScanReport( source_available=False, complete=False, - errors=(message,), + errors=tuple(errors), ) - logger.debug("Found %d manifest file(s)", len(manifest_files)) - for manifest_path in manifest_files: + v2_entries = entries.get(V2_LAYOUT, {}) + legacy_entries = entries.get(LEGACY_LAYOUT, {}) + selected: list[_ManifestEntry] = [] + for key in sorted(v2_entries.keys() | legacy_entries.keys()): + v2 = v2_entries.get(key) + legacy = legacy_entries.get(key) + # Like Ollama, prefer manifests-v2 and fall back to the legacy + # tree only when the v2 name is absent or a dangling link. + if v2 is not None and (legacy is None or v2.path.exists()): + selected.append(v2) + elif legacy is not None: + selected.append(legacy) + + models: list[OllamaModel] = [] + for entry in selected: try: - model = self._parse_manifest(manifest_path) + model = self._parse_manifest(entry) except (OSError, UnicodeError) as exc: - message = f"Could not inspect manifest {manifest_path}: {exc}" + message = f"Could not inspect manifest {entry.path}: {exc}" logger.warning(message) errors.append(message) continue - if model is not None: - models.append(model) - logger.debug( - "Parsed model: %s (readiness=%s)", - model.canonical_name, - model.readiness, - ) - else: - logger.debug("Skipped manifest: %s", manifest_path) + models.append(model) + logger.debug( + "Parsed model: %s (readiness=%s, layout=%s)", + model.canonical_name, + model.readiness, + entry.layout, + ) return OllamaScanReport( models=tuple(models), source_available=True, @@ -92,26 +174,150 @@ def scan_error(exc: OSError) -> None: errors=tuple(errors), ) - def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: - logger.debug("Parsing manifest: %s", manifest_path) + @staticmethod + def _enumerate(root: Path) -> list[Path]: + def scan_error(exc: OSError) -> None: + raise exc + + # os.walk lists symlinks (v2 names are links to manifest blobs) among + # the file names without following directory links. + return sorted( + Path(directory) / name + for directory, _, names in os.walk(root, onerror=scan_error) + for name in names + ) + + @staticmethod + def _entry_for_path(root: Path, path: Path, layout: str) -> _ManifestEntry | None: try: - relative_parts = manifest_path.relative_to( - self.config.ollama_manifests_dir - ).parts + parts = path.relative_to(root).parts except ValueError: - logger.debug("Skipping manifest outside manifests dir: %s", manifest_path) return None + if layout == V2_LAYOUT: + # manifests-v2////, with ":" in the + # host percent-encoded so ports survive on every filesystem. + if len(parts) != 4: + return None + registry = re.sub("%3[aA]", ":", parts[0]) + repository = parts[2] + else: + if len(parts) < 4: + return None + registry = parts[0] + repository = "/".join(parts[2:-1]) + if registry.lower() in PUBLIC_REGISTRY_HOSTS: + # One identity for the public registry keeps canonical names, + # LM Studio repos, and sync records stable across layouts. + registry = DEFAULT_REGISTRY + return _ManifestEntry( + registry=registry, + namespace=parts[1], + repository=repository, + tag=parts[-1], + path=path, + layout=layout, + ) - if len(relative_parts) < 4: - logger.debug( - "Skipping manifest with insufficient path parts: %s", manifest_path - ) - return None + def _read_manifest(self, path: Path) -> object: + """Return the parsed manifest document at ``path``. + + Raises ``_ManifestBlobMissing`` for a dangling v2 link and + ``ValueError`` (via ``json``) for malformed content. + """ + try: + data = path.read_bytes() + except FileNotFoundError: + if path.is_symlink(): + raise _ManifestBlobMissing from None + raise + if path.is_symlink(): + match = BLOB_FILENAME_PATTERN.fullmatch(path.resolve().name) + if match is None or hashlib.sha256(data).hexdigest() != ( + match.group(1).lower() + ): + raise ValueError("manifest link does not match its content digest") + return json.loads(data.decode("utf-8")) + + def _read_manifest_blob(self, digest: str) -> object: + blob_path = self.config.ollama_blobs_dir / digest.replace(":", "-") + try: + data = blob_path.read_bytes() + except FileNotFoundError: + raise _ManifestBlobMissing from None + if "sha256:" + hashlib.sha256(data).hexdigest() != digest.lower(): + raise ValueError("child manifest blob does not match its digest") + return json.loads(data.decode("utf-8")) + + def _select_child(self, manifest: dict[str, object]) -> tuple[object, list[str]]: + """Pick the GGUF child of an Ollama manifest list for LM Studio. + + Returns the child document, or ``None`` with the issues explaining why + no importable child exists. + """ + children = manifest.get("manifests") + if not isinstance(children, list): + return None, ["manifest list entries must be a list"] + candidates = [ + child + for child in children + if isinstance(child, dict) + and child.get("mediaType") in (None, "", MANIFEST_MEDIA_TYPE) + ] + missing = False + problems: list[str] = [] + for runner in GGUF_RUNNER_PREFERENCE: + for child in candidates: + if str(child.get("runner", "")).strip().lower() != runner: + continue + if str(child.get("format", "gguf")).strip().lower() != "gguf": + continue + try: + resolved = self._resolve_child(child) + except _ManifestBlobMissing: + missing = True + continue + except (ValueError, UnicodeError) as exc: + problems.append(f"invalid {runner} child manifest: {exc}") + continue + if isinstance(resolved, dict) and ( + resolved.get("mediaType") == MANIFEST_LIST_MEDIA_TYPE + ): + problems.append("nested Ollama manifest lists are not supported") + continue + return resolved, [] + if problems: + return None, problems + if missing: + return None, [ + "GGUF child manifest blob is missing from the Ollama blob store" + ] + if any(self._is_mlx(child) for child in candidates): + return None, [MLX_ONLY_ISSUE] + return None, ["Ollama manifest lists without a GGUF child are not supported"] + + def _resolve_child(self, child: dict[str, object]) -> object: + layers = child.get("layers") + digest = child.get("digest") + if layers is None and isinstance(digest, str) and digest: + if SHA256_DIGEST_PATTERN.fullmatch(digest) is None: + raise ValueError("child manifest has an invalid SHA-256 digest") + return self._read_manifest_blob(digest) + return child - registry = relative_parts[0] - namespace = relative_parts[1] - repository = "/".join(relative_parts[2:-1]) - tag = relative_parts[-1] + @staticmethod + def _is_mlx(manifest: dict[str, object]) -> bool: + return ( + str(manifest.get("runner", "")).strip().lower() == MLX_RUNNER + or str(manifest.get("format", "")).strip().lower() == SAFETENSORS_FORMAT + ) + + def _parse_manifest(self, entry: _ManifestEntry) -> OllamaModel: + manifest_path = entry.path + logger.debug("Parsing manifest: %s", manifest_path) + registry = entry.registry + namespace = entry.namespace + repository = entry.repository + tag = entry.tag logger.debug( "Manifest parsed: registry=%s, namespace=%s, repository=%s, tag=%s", registry, @@ -121,16 +327,45 @@ def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: ) issues: list[str] = [] + # Selection issues explain the whole model; skip layer diagnostics. + layer_issues = True try: - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - logger.debug("Invalid manifest JSON in %s: %s", manifest_path, exc) - issues.append(f"invalid manifest JSON: {exc}") + manifest = self._read_manifest(manifest_path) + except _ManifestBlobMissing: + issues.append("manifest blob is missing from the Ollama blob store") + manifest = {} + layer_issues = False + except UnicodeError: + # Undecodable bytes are a scan error, not a model verdict. + raise + except ValueError as exc: + # json.JSONDecodeError is a ValueError, as is a digest mismatch. + logger.debug("Invalid manifest in %s: %s", manifest_path, exc) + issues.append( + f"invalid manifest JSON: {exc}" + if isinstance(exc, json.JSONDecodeError) + else str(exc) + ) manifest = {} if not isinstance(manifest, dict): logger.debug("Manifest is not a JSON object: %s", manifest_path) issues.append("manifest is not a JSON object") manifest = {} + if manifest.get("mediaType") == MANIFEST_LIST_MEDIA_TYPE: + child, selection_issues = self._select_child(manifest) + issues.extend(selection_issues) + if isinstance(child, dict): + manifest = child + elif child is None: + manifest = {} + layer_issues = False + else: + issues.append("child manifest is not a JSON object") + manifest = {} + layer_issues = False + elif self._is_mlx(manifest): + issues.append(MLX_ONLY_ISSUE) + layer_issues = False layers = manifest.get("layers", []) if not isinstance(layers, list): @@ -154,14 +389,15 @@ def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: projector_count = sum( role is ArtifactRole.PROJECTOR for _, _, role in artifact_layers ) - if model_count == 0: - issues.append("missing Ollama model layer") - elif model_count > 1: - issues.append("multiple Ollama model layers are not supported") - if projector_count > 1: - issues.append("multiple Ollama projector layers are not supported") - for role in sorted(unsupported_roles): - issues.append(f"Ollama {role} layers are not supported") + if layer_issues: + if model_count == 0: + issues.append("missing Ollama model layer") + elif model_count > 1: + issues.append("multiple Ollama model layers are not supported") + if projector_count > 1: + issues.append("multiple Ollama projector layers are not supported") + for role in sorted(unsupported_roles): + issues.append(f"Ollama {role} layers are not supported") # Import order is semantic: partial sync records must always contain # the model artifact, so normalize model layers before projectors @@ -193,7 +429,8 @@ def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: declared_size=primary.declared_size if primary else None, blob_size=primary.blob_size if primary else None, gguf_valid=( - bool(artifacts) + layer_issues + and bool(artifacts) and model_count == 1 and projector_count <= 1 and not unsupported_roles @@ -274,6 +511,6 @@ def _canonical_name( if namespace == "library" else f"{namespace}/{repository}:{tag}" ) - if registry == "registry.ollama.ai": + if registry == DEFAULT_REGISTRY: return base_name return f"{registry}/{base_name}" diff --git a/src/studiolink/service.py b/src/studiolink/service.py index 7a65367..1cd416f 100644 --- a/src/studiolink/service.py +++ b/src/studiolink/service.py @@ -71,7 +71,9 @@ def scan(self) -> list[OllamaModel]: def scan_report(self) -> OllamaScanReport: logger.debug( - "Scanning Ollama manifests at %s", self.config.ollama_manifests_dir + "Scanning Ollama manifests at %s and %s", + self.config.ollama_manifests_v2_path, + self.config.ollama_manifests_dir, ) report = self.ollama.scan_report() logger.debug("Discovered %d model(s)", len(report.models)) @@ -344,8 +346,10 @@ def doctor(self) -> list[DoctorCheck]: ), DoctorCheck( "ollama manifests dir", - self.config.ollama_manifests_dir.exists(), - str(self.config.ollama_manifests_dir), + self.config.ollama_manifests_v2_path.is_dir() + or self.config.ollama_manifests_dir.is_dir(), + f"{self.config.ollama_manifests_v2_path} (Ollama 0.40+), " + f"{self.config.ollama_manifests_dir} (legacy)", ), DoctorCheck( "ollama blobs dir", diff --git a/tests/test_config.py b/tests/test_config.py index 66513b1..26bf74a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -13,6 +13,7 @@ def test_studiolink_models_dir_override(self, monkeypatch, tmp_path): config = StudioLinkConfig.from_env() assert config.ollama_models_dir == custom assert config.ollama_manifests_dir == custom / "manifests" + assert config.ollama_manifests_v2_path == custom / "manifests-v2" assert config.ollama_blobs_dir == custom / "blobs" def test_state_dir_override_derives_paths(self, monkeypatch, tmp_path): diff --git a/tests/test_ollama_adapter.py b/tests/test_ollama_adapter.py index 254054b..2427008 100644 --- a/tests/test_ollama_adapter.py +++ b/tests/test_ollama_adapter.py @@ -189,14 +189,14 @@ def test_io_failure_is_isolated_and_marks_scan_incomplete( broken = write_manifest(config, repository="broken", digest=DIGEST_A) write_manifest(config, repository="healthy", digest=DIGEST_A) write_blob(config, DIGEST_A) - real_read_text = Path.read_text + real_read_bytes = Path.read_bytes - def flaky_read_text(path, *args, **kwargs): + def flaky_read_bytes(path, *args, **kwargs): if path == broken: raise OSError("access denied") - return real_read_text(path, *args, **kwargs) + return real_read_bytes(path, *args, **kwargs) - monkeypatch.setattr(Path, "read_text", flaky_read_text) + monkeypatch.setattr(Path, "read_bytes", flaky_read_bytes) report = OllamaAdapter(config).scan_report() diff --git a/tests/test_ollama_manifests_v2.py b/tests/test_ollama_manifests_v2.py new file mode 100644 index 0000000..1ce7722 --- /dev/null +++ b/tests/test_ollama_manifests_v2.py @@ -0,0 +1,373 @@ +"""Ollama 0.40+ ``manifests-v2`` layout and manifest-list selection. + +The fixtures mirror what Ollama itself writes: manifest JSON is stored as a +content-addressed blob and ``manifests-v2////`` is a +relative symlink to it (a regular-file copy on Windows). Manifest lists hold +per-runner children that reference further manifest blobs. +""" + +from __future__ import annotations + +import hashlib +import json +import os + +import pytest +from conftest import ( + DIGEST_A, + DIGEST_B, + MODEL_MEDIA_TYPE, + make_sync_record, + write_blob, + write_manifest, +) + +from studiolink.config import StudioLinkConfig +from studiolink.models import ModelReadiness +from studiolink.ollama_adapter import ( + MANIFEST_LIST_MEDIA_TYPE, + MANIFEST_MEDIA_TYPE, + MLX_ONLY_ISSUE, + OllamaAdapter, +) +from studiolink.service import StudioLinkService +from studiolink.state import StateStore + +DIGEST_C = "sha256:" + "c" * 64 +TENSOR_MEDIA_TYPE = "application/vnd.ollama.image.tensor" + + +def gguf_manifest(digest: str, **extra: object) -> dict[str, object]: + return { + "schemaVersion": 2, + "mediaType": MANIFEST_MEDIA_TYPE, + "layers": [{"mediaType": MODEL_MEDIA_TYPE, "digest": digest, "size": 12}], + **extra, + } + + +def store_manifest_blob(config: StudioLinkConfig, document: object) -> str: + data = json.dumps(document).encode("utf-8") + digest = "sha256:" + hashlib.sha256(data).hexdigest() + config.ollama_blobs_dir.mkdir(parents=True, exist_ok=True) + (config.ollama_blobs_dir / digest.replace(":", "-")).write_bytes(data) + return digest + + +def name_v2( + config: StudioLinkConfig, + digest: str, + *, + host: str = "ollama.com", + namespace: str = "library", + repository: str = "llama3", + tag: str = "1b", + symlink: bool = True, +): + path = config.ollama_manifests_v2_path / host / namespace / repository / tag + path.parent.mkdir(parents=True, exist_ok=True) + blob = config.ollama_blobs_dir / digest.replace(":", "-") + if not symlink: + path.write_bytes(blob.read_bytes()) + return path + try: + os.symlink(os.path.relpath(blob, path.parent), path) + except (OSError, NotImplementedError) as exc: # pragma: no cover - Windows + pytest.skip(f"symlinks unavailable: {exc}") + return path + + +def write_v2(config: StudioLinkConfig, document: object, **name: object): + return name_v2(config, store_manifest_blob(config, document), **name) + + +def scan(config: StudioLinkConfig): + return OllamaAdapter(config).scan_report() + + +def only_model(config: StudioLinkConfig): + report = scan(config) + assert report.complete, report.errors + assert len(report.models) == 1, report.models + return report.models[0] + + +class TestV2Names: + def test_v2_only_model_is_discovered_with_legacy_identity(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + path = write_v2(config, gguf_manifest(DIGEST_A)) + + model = only_model(config) + + assert model.readiness is ModelReadiness.READY + assert model.canonical_name == "llama3:1b" + assert model.registry == "registry.ollama.ai" + assert model.fully_qualified_name == "registry.ollama.ai/library/llama3:1b" + assert model.user_repo == "ollama/llama3" + assert model.manifest_path == path + assert model.model_digest == DIGEST_A + + def test_regular_file_copy_is_supported(self, make_config): + # Windows (0.40.1+) copies the manifest blob instead of linking it. + config = make_config() + write_blob(config, DIGEST_A) + write_v2(config, gguf_manifest(DIGEST_A), symlink=False) + + assert only_model(config).readiness is ModelReadiness.READY + + def test_v2_wins_over_legacy_for_the_same_name(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_blob(config, DIGEST_B) + write_manifest(config, digest=DIGEST_A) + write_v2(config, gguf_manifest(DIGEST_B)) + + model = only_model(config) + + assert model.model_digest == DIGEST_B + + def test_dangling_v2_link_falls_back_to_legacy(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_manifest(config, digest=DIGEST_A) + path = write_v2(config, gguf_manifest(DIGEST_B)) + os.remove(os.path.realpath(path)) + + model = only_model(config) + + assert model.model_digest == DIGEST_A + assert model.readiness is ModelReadiness.READY + + def test_dangling_v2_link_alone_is_stale_not_hidden(self, make_config): + config = make_config() + path = write_v2(config, gguf_manifest(DIGEST_A)) + os.remove(os.path.realpath(path)) + + model = only_model(config) + + assert model.canonical_name == "llama3:1b" + assert model.readiness is ModelReadiness.STALE + + def test_legacy_and_v2_models_are_merged_and_sorted(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_manifest(config, repository="zeta", digest=DIGEST_A) + write_v2(config, gguf_manifest(DIGEST_A), repository="alpha") + + names = [m.canonical_name for m in scan(config).models] + + assert names == ["alpha:1b", "zeta:1b"] + + def test_v2_only_library_is_available(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_v2(config, gguf_manifest(DIGEST_A)) + + report = scan(config) + + assert not config.ollama_manifests_dir.exists() + assert report.source_available is True + + def test_encoded_port_in_host_is_decoded(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_v2( + config, + gguf_manifest(DIGEST_A), + host="localhost%3A5000", + namespace="team", + repository="coder", + ) + + model = only_model(config) + + assert model.registry == "localhost:5000" + assert model.canonical_name == "localhost:5000/team/coder:1b" + + def test_converted_legacy_shadow_tags_are_ignored(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_manifest(config, digest=DIGEST_A) + write_manifest(config, repository="llamacpp", tag="d" * 64, digest=DIGEST_A) + + assert only_model(config).canonical_name == "llama3:1b" + + def test_legacy_downgrade_anchor_still_parses(self, make_config): + # Ollama appends manifest/list references to the legacy entry it + # keeps for downgrades; they are not model artifacts. + config = make_config() + write_blob(config, DIGEST_A) + anchor = gguf_manifest(DIGEST_A) + anchor["layers"] += [ + {"mediaType": MANIFEST_LIST_MEDIA_TYPE, "digest": DIGEST_B, "size": 1}, + {"mediaType": MANIFEST_MEDIA_TYPE, "digest": DIGEST_C, "size": 1}, + ] + write_manifest(config, content=anchor) + + assert only_model(config).readiness is ModelReadiness.READY + + def test_link_whose_content_does_not_match_its_digest_is_invalid(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + path = write_v2(config, gguf_manifest(DIGEST_A)) + with open(os.path.realpath(path), "a", encoding="utf-8") as handle: + handle.write(" ") + + model = only_model(config) + + assert model.readiness is ModelReadiness.INVALID + assert "manifest link does not match its content digest" in model.issues + + +def child_ref(config, document, runner, fmt="gguf"): + return { + "mediaType": MANIFEST_MEDIA_TYPE, + "digest": store_manifest_blob(config, document), + "runner": runner, + "format": fmt, + } + + +def manifest_list(*children): + return { + "schemaVersion": 2, + "mediaType": MANIFEST_LIST_MEDIA_TYPE, + "manifests": list(children), + } + + +class TestManifestLists: + def test_prefers_llamacpp_child_over_ggml(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_blob(config, DIGEST_B) + write_v2( + config, + manifest_list( + child_ref(config, gguf_manifest(DIGEST_A), "ggml"), + child_ref(config, gguf_manifest(DIGEST_B), "llamacpp"), + ), + ) + + model = only_model(config) + + assert model.readiness is ModelReadiness.READY + assert model.model_digest == DIGEST_B + + def test_uses_ggml_child_when_it_is_the_only_gguf(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + write_v2( + config, + manifest_list(child_ref(config, gguf_manifest(DIGEST_A), "ggml")), + ) + + assert only_model(config).model_digest == DIGEST_A + + def test_falls_back_when_preferred_child_blob_is_missing(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + missing = child_ref(config, gguf_manifest(DIGEST_B), "llamacpp") + (config.ollama_blobs_dir / missing["digest"].replace(":", "-")).unlink() + write_v2( + config, + manifest_list(missing, child_ref(config, gguf_manifest(DIGEST_A), "ggml")), + ) + + assert only_model(config).model_digest == DIGEST_A + + def test_inline_children_are_supported(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + child = gguf_manifest(DIGEST_A, runner="llamacpp", format="gguf") + write_v2(config, manifest_list(child)) + + assert only_model(config).model_digest == DIGEST_A + + def test_missing_gguf_child_blob_is_stale(self, make_config): + config = make_config() + child = child_ref(config, gguf_manifest(DIGEST_A), "llamacpp") + (config.ollama_blobs_dir / child["digest"].replace(":", "-")).unlink() + write_v2(config, manifest_list(child)) + + model = only_model(config) + + assert model.readiness is ModelReadiness.STALE + + def test_mlx_only_list_is_reported_clearly(self, make_config): + config = make_config() + mlx = { + "schemaVersion": 2, + "mediaType": MANIFEST_MEDIA_TYPE, + "layers": [{"mediaType": TENSOR_MEDIA_TYPE, "digest": DIGEST_A}], + } + write_v2(config, manifest_list(child_ref(config, mlx, "mlx", "safetensors"))) + + model = only_model(config) + + assert model.readiness is ModelReadiness.INVALID + assert model.issues == (MLX_ONLY_ISSUE,) + + def test_plain_mlx_manifest_is_reported_clearly(self, make_config): + config = make_config() + write_v2( + config, + { + "schemaVersion": 2, + "mediaType": MANIFEST_MEDIA_TYPE, + "runner": "mlx", + "format": "safetensors", + "layers": [{"mediaType": TENSOR_MEDIA_TYPE, "digest": DIGEST_A}], + }, + ) + + model = only_model(config) + + assert model.readiness is ModelReadiness.INVALID + assert model.issues == (MLX_ONLY_ISSUE,) + + def test_child_blob_with_wrong_content_is_invalid(self, make_config): + config = make_config() + write_blob(config, DIGEST_A) + child = child_ref(config, gguf_manifest(DIGEST_A), "llamacpp") + blob = config.ollama_blobs_dir / child["digest"].replace(":", "-") + blob.write_bytes(blob.read_bytes() + b" ") + write_v2(config, manifest_list(child)) + + model = only_model(config) + + assert model.readiness is ModelReadiness.INVALID + assert any("does not match its digest" in issue for issue in model.issues) + + +class TestV2Lifecycle: + def test_prune_keeps_import_for_model_only_in_v2(self, make_config, fake_lmstudio): + # Regression: models that exist only in manifests-v2 (e.g. after + # `ollama cp`) looked deleted, so prune removed their LM Studio import. + config = make_config() + config.lmstudio_models_dir.mkdir(parents=True) + write_blob(config, DIGEST_A) + write_v2(config, gguf_manifest(DIGEST_A)) + store = StateStore(config.state_file) + store.upsert(make_sync_record("llama3:1b", DIGEST_A)) + service = StudioLinkService( + config=config, ollama=OllamaAdapter(config), lmstudio=fake_lmstudio + ) + + report = service.prune() + + assert report.records_removed == () + assert set(store.get_all_records()) == {"llama3:1b"} + + def test_doctor_accepts_a_v2_only_library(self, make_config, fake_lmstudio): + config = make_config() + config.ollama_manifests_v2_path.mkdir(parents=True) + service = StudioLinkService( + config=config, ollama=OllamaAdapter(config), lmstudio=fake_lmstudio + ) + + check = next(c for c in service.doctor() if c.name == "ollama manifests dir") + + assert check.ok is True + assert "manifests-v2" in check.details diff --git a/tests/test_service.py b/tests/test_service.py index 01607b9..8ed0283 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -140,14 +140,14 @@ def test_refuses_prune_after_partial_manifest_scan( config.import_staging_dir.mkdir(parents=True) alias = config.import_staging_dir / "orphan.gguf" alias.write_bytes(b"GGUFxxxx") - real_read_text = Path.read_text + real_read_bytes = Path.read_bytes - def flaky_read_text(path, *args, **kwargs): + def flaky_read_bytes(path, *args, **kwargs): if path == broken: raise OSError("access denied") - return real_read_text(path, *args, **kwargs) + return real_read_bytes(path, *args, **kwargs) - monkeypatch.setattr(Path, "read_text", flaky_read_text) + monkeypatch.setattr(Path, "read_bytes", flaky_read_bytes) with pytest.raises(RuntimeError, match="scan was incomplete"): service.prune() From a0800d4cca5c1eacd132200c8671e8f7b40bf79d Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:33:07 +0530 Subject: [PATCH 10/11] fix: keep symlinked imports and hard-link fallback working on Windows On Windows, os.readlink returns absolute targets in \\?\ extended-length form, so symlink_references and owns_destination never matched recorded alias paths. Prune and replacement imports then deleted aliases that LM Studio or user symlinks still depended on, breaking those imports. Normalize link targets through one helper before comparing. publish_import only fell back to a no-overwrite rename for EPERM/ENOTSUP. Windows reports missing hard-link support as EOPNOTSUPP or through winerror 1/50, so imports onto such volumes failed instead. Co-Authored-By: Claude --- src/studiolink/files.py | 22 ++++++++++++++++++---- src/studiolink/transactions.py | 9 ++++++++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/studiolink/files.py b/src/studiolink/files.py index 1f310bc..8aa7101 100644 --- a/src/studiolink/files.py +++ b/src/studiolink/files.py @@ -10,6 +10,22 @@ from studiolink.models import LinkMode, OllamaArtifact, SyncedArtifact, SyncRecord +def link_target(path: Path) -> Path: + """Return the absolute path a symlink points to, without following chains. + + Windows reports absolute targets in their ``\\\\?\\`` extended-length + form, which never compares equal to the paths StudioLink records, so strip + it before resolving the target against the link's directory. + """ + raw = os.readlink(path) + if os.name == "nt": + if raw.startswith("\\\\?\\UNC\\"): + raw = "\\\\" + raw[len("\\\\?\\UNC\\") :] + elif raw.startswith("\\\\?\\"): + raw = raw[len("\\\\?\\") :] + return Path(os.path.abspath(path.parent / raw)) + + def same_file(left: Path, right: Path) -> bool: """Check whether two paths reference the same inode (hard link).""" try: @@ -79,8 +95,7 @@ def owns_destination(record: SyncRecord, artifact: SyncedArtifact, path: Path) - return identity(path) == artifact.imported_identity # Legacy records have no file identity. Require evidence before removal. if path.is_symlink(): - target = path.parent / os.readlink(path) - return target.absolute() == artifact.import_alias_path.absolute() + return link_target(path) == artifact.import_alias_path.absolute() for source in (artifact.import_alias_path, artifact.blob_path): if same_file(path, source): return True @@ -170,6 +185,5 @@ def fail(exc: OSError) -> None: for name in (*dirs, *files): path = Path(directory) / name if path.is_symlink(): - target = Path(os.path.abspath(path.parent / os.readlink(path))) - references.setdefault(target, set()).add(path) + references.setdefault(link_target(path), set()).add(path) return references diff --git a/src/studiolink/transactions.py b/src/studiolink/transactions.py index 7cfee64..5c16d74 100644 --- a/src/studiolink/transactions.py +++ b/src/studiolink/transactions.py @@ -16,6 +16,10 @@ from studiolink.state import StateStore TRANSACTIONS = ".studiolink-transactions" +# errno values meaning "this filesystem cannot hard-link"; Windows reports +# ERROR_INVALID_FUNCTION (1) or ERROR_NOT_SUPPORTED (50) through winerror. +_NO_HARDLINK_ERRNOS = frozenset({errno.EPERM, errno.ENOTSUP, errno.EOPNOTSUPP}) +_NO_HARDLINK_WINERRORS = frozenset({1, 50}) MARKER = "owner.json" @@ -118,7 +122,10 @@ def publish_import(staged: Path, target: Path) -> None: try: os.link(staged, target) except OSError as exc: - if exc.errno not in (errno.EPERM, errno.ENOTSUP): + if ( + exc.errno not in _NO_HARDLINK_ERRNOS + and getattr(exc, "winerror", None) not in _NO_HARDLINK_WINERRORS + ): raise _rename_without_overwrite(staged, target) staged.unlink(missing_ok=True) From b2a2596e1daeba95fb9a7d43a0ca9e4b457ee907 Mon Sep 17 00:00:00 2001 From: Manan <143382097+ensevengg@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:58:30 +0530 Subject: [PATCH 11/11] fix: address review findings on import destinations and prune - Reject destinations whose parent resolves outside the LM Studio models root before publishing, and spell every destination under the configured root so the same file compares equal however it was recorded. - Replace ":" in registry names when building LM Studio repositories; Windows rejects it, and manifests-v2 makes port registries visible there. - Count recovered transactions in dry-run link accounting so a shared inode reports its space once. - Never prune a managed destination that an untracked symlink still depends on; keep its record instead. - Confirm a replacement vision bundle before reclaiming the previous one, keeping the old import and record when LM Studio rejects the new bundle. - Share the no-hard-link error classification with copy reuse so it falls back to a normal copy instead of failing. Co-Authored-By: Claude --- src/studiolink/files.py | 41 ++++--- src/studiolink/models.py | 3 +- src/studiolink/service.py | 14 ++- src/studiolink/syncer.py | 51 +++++--- src/studiolink/transactions.py | 15 ++- tests/test_review_fixes.py | 205 +++++++++++++++++++++++++++++++++ 6 files changed, 294 insertions(+), 35 deletions(-) create mode 100644 tests/test_review_fixes.py diff --git a/src/studiolink/files.py b/src/studiolink/files.py index 8aa7101..40cfa5f 100644 --- a/src/studiolink/files.py +++ b/src/studiolink/files.py @@ -62,29 +62,42 @@ def equal_contents(left: Path, right: Path) -> bool: return False +def _contained(root: Path, path: Path) -> Path: + """Spell ``path`` under the configured root, rejecting escapes. + + Only the parent is resolved: the imported file itself may be a symlink to + Ollama. Re-expressing the resolved parent under ``root`` gives one spelling + per destination even when the root or a directory below it is a symlink, + so destinations compare equal wherever they were recorded. + """ + try: + relative = path.parent.resolve().relative_to(root.resolve()) + except ValueError: + raise ValueError( + f"Import directory resolves outside the configured models directory: {path}" + ) from None + return root.absolute().joinpath(relative, path.name) + + def destination_path(root: Path, user_repo: str, filename: str) -> Path: parts = user_repo.split("/") if len(parts) != 2 or any(p in ("", ".", "..") or "\\" in p for p in parts): raise ValueError(f"Invalid import repository: {user_repo}") - return root.absolute().joinpath(*parts, Path(filename).name) + return _contained(root, root.absolute().joinpath(*parts, Path(filename).name)) def artifact_destination( record: SyncRecord, artifact: SyncedArtifact, root: Path ) -> Path: - path = artifact.imported_model_path or destination_path( - root, record.user_repo, artifact.import_alias_path.name - ) - # Resolve the parent only: the imported file itself may be a symlink to Ollama. - if not path.parent.resolve().is_relative_to(root.resolve()): - # LM Studio's library may have moved since the record was saved. Check - # its expected location under the new root; never delete the old path. - path = destination_path(root, record.user_repo, artifact.import_alias_path.name) - if not path.parent.resolve().is_relative_to(root.resolve()): - raise ValueError( - f"Import directory resolves outside the configured models directory: {path}" - ) - return path + if artifact.imported_model_path is not None: + try: + return _contained(root, artifact.imported_model_path) + except ValueError: + # LM Studio's library may have moved since the record was saved. + # Check its expected location under the new root; never delete + # the old path. + pass + return destination_path(root, record.user_repo, artifact.import_alias_path.name) def owns_destination(record: SyncRecord, artifact: SyncedArtifact, path: Path) -> bool: diff --git a/src/studiolink/models.py b/src/studiolink/models.py index 1cade4e..ccb8026 100644 --- a/src/studiolink/models.py +++ b/src/studiolink/models.py @@ -110,7 +110,8 @@ def short_name(self) -> str: def user_repo(self) -> str: repo_parts = [] if self.registry != "registry.ollama.ai": - repo_parts.append(self.registry.replace(".", "-")) + # ":" (a registry port) is not allowed in Windows directory names. + repo_parts.append(self.registry.replace(".", "-").replace(":", "-")) if self.namespace != "library": repo_parts.append(self.namespace) repo_parts.append(self.repository.replace("/", "--")) diff --git a/src/studiolink/service.py b/src/studiolink/service.py index 1cd416f..9b6483d 100644 --- a/src/studiolink/service.py +++ b/src/studiolink/service.py @@ -210,9 +210,21 @@ def superseded(record: SyncRecord, model: OllamaModel) -> bool: # Simulate link-count changes in dry runs as well, so an inode shared by # several doomed paths is counted exactly once. removed_links: dict[tuple[int, int], int] = {} + if dry_run: + # Recovered transactions are planned removals too; count them so a + # shared inode reports its space exactly once. + for result in recovered: + try: + stat = result.path.lstat() + except OSError: + continue + key = (stat.st_dev, stat.st_ino) + removed_links[key] = removed_links.get(key, 0) + 1 def remove(path, reason, results): - if results is aliases and any( + # Never break a symlink that still depends on this entry, whether + # it is a staging alias or a managed LM Studio destination. + if any( ref.is_symlink() for ref in references.get(path.absolute(), ()) if ref not in seen diff --git a/src/studiolink/syncer.py b/src/studiolink/syncer.py index 568bea7..5eeef77 100644 --- a/src/studiolink/syncer.py +++ b/src/studiolink/syncer.py @@ -39,7 +39,11 @@ ) from studiolink.ports import LMStudioPort from studiolink.state import StateStore -from studiolink.transactions import import_transaction, recover_imports +from studiolink.transactions import ( + hardlinks_unsupported, + import_transaction, + recover_imports, +) logger = logging.getLogger("studiolink") @@ -162,16 +166,19 @@ def _sync_one( ), ) if dry_run: - targets = ", ".join( - str( - destination_path( - root, - model.import_user_repo, - self._source_path(model, artifact, import_mode).name, + try: + targets = ", ".join( + str( + destination_path( + root, + model.import_user_repo, + self._source_path(model, artifact, import_mode).name, + ) ) + for artifact in sources ) - for artifact in sources - ) + except ValueError as exc: + return SyncResult(model=model, status="error", message=str(exc)) return SyncResult( model=model, status="dry-run", @@ -199,6 +206,23 @@ def _sync_one( message = _format_import_message(import_result) record = self._make_record(model, completed, link_mode) + if len(completed) == 1: + result = SyncResult( + model=model, status="synced", message=message, record=record + ) + else: + # Confirm a bundle before reclaiming the one it replaces, so a + # bundle LM Studio rejects never costs the previous working one. + result = self._confirm_bundle(model, record, "synced", message) + if result.status == "error": + if existing is not None and not ( + self._targets(existing) <= self._targets(record) + ): + # Keep the previous import tracked; a retry adopts the + # new files without another copy. + return replace(result, record=None) + return result + record = result.record or record if existing is not None: try: self._reclaim(existing, record, records) @@ -206,11 +230,7 @@ def _sync_one( # Keep the old record so its cleanup is retried; the new import # is verified and will be adopted without another copy. return SyncResult(model=model, status="error", message=str(exc)) - if len(completed) == 1: - return SyncResult( - model=model, status="synced", message=message, record=record - ) - return self._confirm_bundle(model, record, "synced", message) + return result def _sync_artifact( self, @@ -484,7 +504,8 @@ def _reuse_copy( try: os.link(source, target) except OSError as exc: - if exc.errno in (errno.EXDEV, errno.ENOTSUP): + if exc.errno == errno.EXDEV or hardlinks_unsupported(exc): + # The caller falls back to an ordinary copy. continue raise return True diff --git a/src/studiolink/transactions.py b/src/studiolink/transactions.py index 5c16d74..6e4b14d 100644 --- a/src/studiolink/transactions.py +++ b/src/studiolink/transactions.py @@ -20,6 +20,16 @@ # ERROR_INVALID_FUNCTION (1) or ERROR_NOT_SUPPORTED (50) through winerror. _NO_HARDLINK_ERRNOS = frozenset({errno.EPERM, errno.ENOTSUP, errno.EOPNOTSUPP}) _NO_HARDLINK_WINERRORS = frozenset({1, 50}) + + +def hardlinks_unsupported(exc: OSError) -> bool: + """Whether ``os.link`` failed because the filesystem cannot hard-link.""" + return ( + exc.errno in _NO_HARDLINK_ERRNOS + or getattr(exc, "winerror", None) in _NO_HARDLINK_WINERRORS + ) + + MARKER = "owner.json" @@ -122,10 +132,7 @@ def publish_import(staged: Path, target: Path) -> None: try: os.link(staged, target) except OSError as exc: - if ( - exc.errno not in _NO_HARDLINK_ERRNOS - and getattr(exc, "winerror", None) not in _NO_HARDLINK_WINERRORS - ): + if not hardlinks_unsupported(exc): raise _rename_without_overwrite(staged, target) staged.unlink(missing_ok=True) diff --git a/tests/test_review_fixes.py b/tests/test_review_fixes.py new file mode 100644 index 0000000..93aae67 --- /dev/null +++ b/tests/test_review_fixes.py @@ -0,0 +1,205 @@ +"""Regression tests for PR #5 review findings.""" + +from __future__ import annotations + +import errno +import json +import os +from pathlib import Path + +import pytest +from conftest import ( + DIGEST_A, + DIGEST_B, + make_sync_record, + write_blob, + write_manifest, +) +from test_import_lifecycle import destination + +import studiolink.syncer as syncer_module +from studiolink.files import artifact_destination, destination_path +from studiolink.models import LinkMode, SyncedArtifact +from studiolink.state import StateStore +from studiolink.transactions import TRANSACTIONS + +PROJECTOR_DIGEST = "sha256:" + "c" * 64 + + +def write_bundle(config, model_digest): + write_manifest( + config, + repository="llava", + content={ + "layers": [ + { + "mediaType": "application/vnd.ollama.image.model", + "digest": model_digest, + }, + { + "mediaType": "application/vnd.ollama.image.projector", + "digest": PROJECTOR_DIGEST, + }, + ] + }, + ) + write_blob(config, model_digest) + write_blob(config, PROJECTOR_DIGEST) + + +def symlink_or_skip(link: Path, target: Path, **kwargs) -> None: + try: + link.symlink_to(target, **kwargs) + except (OSError, NotImplementedError) as exc: # pragma: no cover - Windows + pytest.skip(f"symlinks unavailable: {exc}") + + +def test_sync_rejects_repository_directory_escaping_models_root(library, tmp_path): + config, service, adapter, _, _ = library + outside = tmp_path / "elsewhere" + outside.mkdir() + config.lmstudio_models_dir.mkdir(parents=True) + symlink_or_skip( + config.lmstudio_models_dir / "ollama", outside, target_is_directory=True + ) + + result = service.sync(sync_all=True)[0] + + assert result.status == "error" + assert "outside the configured models directory" in result.message + assert adapter.calls == [] + assert not list(outside.iterdir()) + assert StateStore(config.state_file).get_all_records() == {} + + +def test_sync_dry_run_reports_escaping_destination(library, tmp_path): + config, service, _, _, _ = library + outside = tmp_path / "elsewhere" + outside.mkdir() + config.lmstudio_models_dir.mkdir(parents=True) + symlink_or_skip( + config.lmstudio_models_dir / "ollama", outside, target_is_directory=True + ) + + result = service.sync(sync_all=True, dry_run=True)[0] + + assert result.status == "error" + + +def test_destinations_share_one_spelling_through_a_symlinked_root(tmp_path): + real = tmp_path / "real-models" + real.mkdir() + alias_root = tmp_path / "models" + symlink_or_skip(alias_root, real, target_is_directory=True) + recorded = real / "ollama" / "llama3" / "weights.gguf" + record = make_sync_record() + artifact = SyncedArtifact( + role=record.primary_artifact.role, + digest=DIGEST_A, + blob_path=Path("blob"), + import_alias_path=Path("weights.gguf"), + imported_model_path=recorded, + ) + + spelled = artifact_destination(record, artifact, alias_root) + + assert spelled == alias_root.absolute() / "ollama" / "llama3" / "weights.gguf" + assert spelled == destination_path(alias_root, "ollama/llama3", "weights.gguf") + + +def test_registry_port_never_reaches_repository_name(): + from conftest import make_ollama_model + + model = make_ollama_model( + "localhost:5000/team/coder:1b", + registry="localhost:5000", + namespace="team", + repository="coder", + tag="1b", + ) + + assert ":" not in model.user_repo + assert model.user_repo == "ollama/localhost-5000-team-coder" + + +def test_dry_run_counts_inode_shared_with_recovered_transaction_once(library): + config, service, _, _, _ = library + work = config.lmstudio_models_dir / TRANSACTIONS / "import-interrupted" + work.mkdir(parents=True) + partial = work / "model.gguf" + partial.write_bytes(b"GGUFpartial") + (work / "owner.json").write_text(json.dumps({"filename": partial.name})) + config.import_staging_dir.mkdir(parents=True) + orphan = config.import_staging_dir / "gone-1b-000000000000.gguf" + os.link(partial, orphan) + + report = service.prune(dry_run=True) + + entries = [*report.imports, *report.aliases] + assert {entry.path for entry in entries} == {partial, orphan} + assert sum(entry.would_free for entry in entries) == 1 + + +def test_prune_keeps_destination_an_untracked_symlink_depends_on(library): + config, service, _, manifest, blob = library + service.sync(sync_all=True) + target = destination(config, service.scan()[0]) + manual = config.lmstudio_models_dir / "manual" / "model" / "saved.gguf" + manual.parent.mkdir(parents=True) + symlink_or_skip(manual, target) + manifest.unlink() + blob.unlink() + + report = service.prune() + + assert target.is_file() + assert manual.read_bytes() == b"GGUF" + b"0" * 8 + assert report.records_removed == () + assert set(StateStore(config.state_file).get_all_records()) == {"llama3:1b"} + + +def test_rejected_replacement_bundle_keeps_previous_bundle(make_syncer, monkeypatch): + from test_syncer import scan_models, sync_kwargs + + syncer, config, fake = make_syncer() + write_bundle(config, DIGEST_A) + first = syncer.sync(scan_models(config), **sync_kwargs())[0] + assert first.status == "synced" + store = StateStore(config.state_file) + previous = store.get_record("llava:1b") + old_targets = [ + artifact_destination(previous, artifact, config.lmstudio_models_dir) + for artifact in previous.artifacts + ] + write_bundle(config, DIGEST_B) + fake.inventory_override = () + monkeypatch.setattr(syncer_module.time, "sleep", lambda _: None) + + result = syncer.sync(scan_models(config), **sync_kwargs())[0] + + assert result.status == "error" + assert "vision-capable" in result.message + assert store.get_record("llava:1b") == previous + assert all(target.is_file() for target in old_targets) + + +def test_copy_reuse_falls_back_when_hardlinks_are_unsupported(make_syncer, monkeypatch): + from test_syncer import scan_models, sync_kwargs + + syncer, config, fake = make_syncer() + write_manifest(config, tag="1b", digest=DIGEST_A) + write_manifest(config, tag="latest", digest=DIGEST_A) + write_blob(config, DIGEST_A) + real_link = os.link + + def no_hardlinks_in_lmstudio(source, target, *args, **kwargs): + if Path(target).is_relative_to(config.lmstudio_models_dir): + raise OSError(errno.EPERM, "hard links not permitted") + return real_link(source, target, *args, **kwargs) + + monkeypatch.setattr(syncer_module.os, "link", no_hardlinks_in_lmstudio) + + results = syncer.sync(scan_models(config), **sync_kwargs(link_mode=LinkMode.COPY)) + + assert [result.status for result in results] == ["synced", "synced"] + assert len(fake.calls) == 2