From 3d5c60b1bc6b3927b1fb2be196d0fd87c591526c Mon Sep 17 00:00:00 2001 From: Manan JK Date: Mon, 31 Aug 2026 13:31:40 +0530 Subject: [PATCH] fix: harden sync lifecycle and platform discovery --- .github/workflows/ci.yml | 20 ++- README.md | 49 +++---- pyproject.toml | 31 ++++- src/studiolink/__init__.py | 2 +- src/studiolink/__main__.py | 2 +- src/studiolink/cli.py | 12 +- src/studiolink/config.py | 33 ++++- src/studiolink/lmstudio_adapter.py | 12 +- src/studiolink/models.py | 38 +++++- src/studiolink/ollama_adapter.py | 64 +++++++-- src/studiolink/ports.py | 15 ++- src/studiolink/service.py | 121 +++++++++++------ src/studiolink/state.py | 10 +- src/studiolink/syncer.py | 63 ++++++--- tests/conftest.py | 21 ++- tests/test_cli.py | 28 ++-- tests/test_config.py | 90 +++++++++++-- tests/test_lmstudio_adapter.py | 23 ++++ tests/test_models.py | 8 +- tests/test_ollama_adapter.py | 44 +++++++ tests/test_service.py | 200 +++++++++++++++++++++++++++-- tests/test_state.py | 13 +- tests/test_syncer.py | 58 ++++++++- 23 files changed, 771 insertions(+), 186 deletions(-) create mode 100644 tests/test_lmstudio_adapter.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3555af4..ee3273a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,30 @@ on: pull_request: jobs: + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install + run: pip install -e ".[dev]" + - name: Lint + run: ruff check src tests + - name: Format + run: ruff format --check src tests + - name: Type check + run: pyright + - name: Build distributions + run: python -m build + test: strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ["3.12", "3.13"] + python-version: ["3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 051334c..6375dba 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ StudioLink is a Python CLI tool that bridges Ollama and LM Studio. It treats Oll - ๐Ÿ”„ **Sync** - Import models into LM Studio with one command - ๐Ÿ“Š **Status** - Track which models are synced and their current state - ๐Ÿฉบ **Doctor** - Verify prerequisites and diagnose issues -- ๐Ÿงน **Prune** - Reclaim disk space pinned by orphaned import aliases +- ๐Ÿงน **Prune** - Safely remove orphaned aliases without breaking symbolic imports - ๐Ÿ”— **Smart linking** - Uses hard links by default; pass `--copy` when volumes differ - ๐Ÿ’พ **State tracking** - Remembers what's been synced to avoid re-importing @@ -18,7 +18,7 @@ StudioLink is a Python CLI tool that bridges Ollama and LM Studio. It treats Oll Before using StudioLink, ensure you have: -1. **Ollama** installed, with its model library at `~/.ollama/models/` (or wherever `OLLAMA_MODELS` points) +1. **Ollama** installed with a local model library (StudioLink detects current platform defaults and honors `OLLAMA_MODELS`) 2. **LM Studio** with the `lms` CLI installed 3. **Python 3.12+** @@ -100,6 +100,9 @@ sdl sync --hard-link sdl sync --copy sdl sync --symbolic-link +# Import the opaque Ollama blob path directly (advanced) +sdl sync --direct + # Verbose output sdl -v sync --all ``` @@ -110,6 +113,7 @@ sdl -v sync --all - `--copy` - Use copy mode instead of hard links - `--hard-link` - Force hard link mode (default) - `--symbolic-link` - Use symbolic links +- `--direct` - Import the Ollama blob directly without a human-readable alias ### `sdl status` Show discovered models and their sync state. @@ -131,9 +135,9 @@ Discovered 3 model(s); 1 tracked as synced. ``` ### `sdl prune` -Remove staging aliases and sync state for models that Ollama no longer has. +Remove aliases and sync state that are no longer needed, after first verifying that the Ollama library was scanned completely. -Because synced models are hard-linked, deleting a model in Ollama does **not** free its disk space until the matching import alias is removed - `sdl prune` does exactly that. +Space recovery depends on the import mode. A copied import can leave the staging alias as the last reference to Ollama's former blob, while an LM Studio hard link continues to retain the model data after the staging alias is removed. Aliases required by symbolic-link imports are preserved so LM Studio is not left with a broken link. ```powershell # Preview what would be removed @@ -159,7 +163,7 @@ sdl doctor --json - LM Studio CLI (`lms`) exists - Ollama manifests and blobs directories exist - LM Studio models directory exists -- Hard-link volume compatibility (same drive) +- Hard-link filesystem/volume compatibility - Discovered Ollama models - Blob presence validation - GGUF header validation @@ -202,23 +206,24 @@ When scanning or checking status, models can have these states: Default paths are platform-aware; every one can be overridden with an environment variable: -| Setting | Windows default | Linux/macOS default | Environment Variable | -|---------|-----------------|---------------------|---------------------| -| Ollama executable | `~\AppData\Local\Programs\Ollama\ollama.exe` | `ollama` from `PATH` (fallback `/usr/bin/ollama`) | `STUDIOLINK_OLLAMA_EXE` | -| LM Studio CLI | `~\.lmstudio\bin\lms.exe` | `lms` from `PATH` (fallback `~/.lmstudio/bin/lms`) | `STUDIOLINK_LMS_EXE` | -| Ollama models | `~\.ollama\models` | `~/.ollama/models` | `STUDIOLINK_OLLAMA_MODELS_DIR` (falls back to Ollama's own `OLLAMA_MODELS`) | -| LM Studio models | `~\.lmstudio\models` | `~/.lmstudio/models` | `STUDIOLINK_LMSTUDIO_MODELS_DIR` | -| State directory | `~\.studiolink` | `~/.studiolink` | `STUDIOLINK_STATE_DIR` | +| Setting | Windows default | Linux default | macOS default | Environment Variable | +|---------|-----------------|---------------|---------------|---------------------| +| Ollama executable | `ollama` from `PATH`, then the standard app path | `ollama` from `PATH` (fallback `/usr/bin/ollama`) | `ollama` from `PATH` (fallback `/usr/bin/ollama`) | `STUDIOLINK_OLLAMA_EXE` | +| LM Studio CLI | `lms` from `PATH`, then `~\.lmstudio\bin\lms.exe` | `lms` from `PATH` (fallback `~/.lmstudio/bin/lms`) | `lms` from `PATH` (fallback `~/.lmstudio/bin/lms`) | `STUDIOLINK_LMS_EXE` | +| Ollama models | `~\.ollama\models` | `/usr/share/ollama/.ollama/models` | `~/.ollama/models` | `STUDIOLINK_OLLAMA_MODELS_DIR` (falls back to Ollama's own `OLLAMA_MODELS`) | +| LM Studio models | LM Studio's configured `downloadsFolder` | LM Studio's configured `downloadsFolder` | LM Studio's configured `downloadsFolder` | `STUDIOLINK_LMSTUDIO_MODELS_DIR` | +| State directory | `~\.studiolink` | `~/.studiolink` | `~/.studiolink` | `STUDIOLINK_STATE_DIR` | ## How It Works -1. **Scanning** - Reads Ollama manifest files from `~/.ollama/models/manifests/` to discover models -2. **Blob Resolution** - Locates GGUF blobs in `~/.ollama/models/blobs/` by digest +1. **Scanning** - Reads manifest files from the configured Ollama models directory +2. **Blob Resolution** - Locates GGUF blobs in Ollama's content-addressed blob directory 3. **Validation** - Verifies blobs start with GGUF magic bytes -4. **Import Aliases** - Creates human-readable `.gguf` hard links in `~/.studiolink/imports/`, named after the model and digest so re-pulled models never reuse a stale alias +4. **Import Aliases** - Creates digest-keyed, human-readable `.gguf` aliases. Copy mode uses a temporary alias on Ollama's filesystem so the state and LM Studio directories may live on other volumes 5. **LM Studio Import** - Uses `lms import` CLI to import models -6. **State Tracking** - Saves sync records to `~/.studiolink/state.json` (written atomically) -7. **Pruning** - `sdl prune` removes aliases for models Ollama no longer has, reclaiming pinned blob space +6. **State Tracking** - Saves sync records and the expected LM Studio target to `~/.studiolink/state.json` (written atomically) +7. **Reconciliation** - Status and sync verify that the expected LM Studio target still exists instead of trusting state alone +8. **Pruning** - Pruning is blocked after unavailable or incomplete Ollama scans and preserves aliases required by symbolic imports ## Troubleshooting @@ -229,16 +234,16 @@ ollama pull ``` ### Hard link fails (cross-volume) -Hard links require the Ollama blob store, the StudioLink state directory, and the LM Studio models directory to live on the same filesystem/volume. If they don't, the sync reports a hard-link error; retry with copy mode instead: +Hard-link imports require the Ollama blob store, StudioLink alias directory, and LM Studio models directory to live on the same filesystem/volume. If they do not, retry with copy mode; copy mode creates its temporary naming alias on Ollama's filesystem and can copy to LM Studio on another volume: ```powershell sdl sync --copy ``` -Alternatively, point `STUDIOLINK_STATE_DIR` at a location on the same volume as your Ollama models. +The StudioLink state directory does not need to share Ollama's volume when using copy mode. ### Deleted a model in Ollama but disk space wasn't freed -Synced models keep their blobs alive through hard links in the StudioLink imports directory. Run `sdl prune` to remove aliases for models Ollama no longer has and reclaim the space. +Run `sdl prune` to remove aliases that are no longer needed. If LM Studio imported the model with `--hard-link`, its final hard link intentionally continues to retain the data; removing only the staging alias cannot free those bytes. Symbolic-link aliases are preserved while LM Studio depends on them. ### LM Studio import fails Run `sdl doctor` to verify: @@ -248,11 +253,11 @@ Run `sdl doctor` to verify: ## Design Notes -- **Ollama as source of truth** - StudioLink never modifies Ollama's model store +- **Ollama as source of truth** - StudioLink never changes Ollama manifests or content-addressed blobs; copy mode may briefly create a `.studiolink-imports` alias directory on the same filesystem - **Hard links by default** - Space-efficient, creates references rather than copies - **No silent fallbacks** - If a hard link is impossible (cross-volume), the sync fails loudly; use `--copy` explicitly - **State persistence** - Tracks imports to avoid redundant operations; state is written atomically and corrupt records are skipped, never crash -- **Prune over leak** - Orphaned aliases are cleaned up on demand with `sdl prune` instead of pinning disk space forever +- **Safe pruning** - Orphaned aliases are cleaned up only after a complete source scan; symbolic-link dependencies are retained - **CLI-only** - No GUI, designed for automation and scripting ## License diff --git a/pyproject.toml b/pyproject.toml index f2b0fa3..e9b2088 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,15 @@ [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] name = "studiolink" -version = "0.2.0" +dynamic = ["version"] description = "Sync Ollama-downloaded GGUF models into LM Studio." readme = "README.md" requires-python = ">=3.12" -license = { file = "LICENSE" } +license = "MIT" +license-files = ["LICENSE"] authors = [ { name = "StudioLink contributors" } ] @@ -17,15 +18,20 @@ classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", "Intended Audience :: End Users/Desktop", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] dependencies = [] [project.optional-dependencies] -dev = ["pytest>=8.0"] +dev = [ + "build>=1.2", + "pyright>=1.1.390", + "pytest>=8.0", + "ruff>=0.9", +] [project.urls] Homepage = "https://github.com/ensevengg/StudioLink" @@ -37,8 +43,23 @@ sdl = "studiolink.cli:main" [tool.setuptools] package-dir = {"" = "src"} +[tool.setuptools.dynamic] +version = {attr = "studiolink.__version__"} + [tool.setuptools.packages.find] where = ["src"] [tool.pytest.ini_options] testpaths = ["tests"] + +[tool.ruff] +target-version = "py312" +line-length = 88 + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] + +[tool.pyright] +include = ["src"] +pythonVersion = "3.12" +typeCheckingMode = "basic" diff --git a/src/studiolink/__init__.py b/src/studiolink/__init__.py index b306482..0a93a73 100644 --- a/src/studiolink/__init__.py +++ b/src/studiolink/__init__.py @@ -2,4 +2,4 @@ __all__ = ["__version__"] -__version__ = "0.2.0" +__version__ = "0.2.1" diff --git a/src/studiolink/__main__.py b/src/studiolink/__main__.py index 191d60b..baf7d8e 100644 --- a/src/studiolink/__main__.py +++ b/src/studiolink/__main__.py @@ -1,6 +1,6 @@ import sys -from studiolink.cli import main +from studiolink.cli import main if __name__ == "__main__": raise SystemExit(main(sys.argv[1:])) diff --git a/src/studiolink/cli.py b/src/studiolink/cli.py index 9f2321f..4119404 100644 --- a/src/studiolink/cli.py +++ b/src/studiolink/cli.py @@ -48,7 +48,9 @@ def build_parser() -> argparse.ArgumentParser: action="version", version=f"StudioLink {__version__}", ) - subparsers = parser.add_subparsers(dest="command", required=True, metavar="") + subparsers = parser.add_subparsers( + dest="command", required=True, metavar="" + ) scan_parser = subparsers.add_parser( "scan", @@ -334,8 +336,8 @@ def run_help(args: argparse.Namespace, service: StudioLinkService) -> int: Usage: sdl scan [--json] [-v] sync Import one or more models into LM Studio. - Usage: sdl sync [...] [--copy|--hard-link|--symbolic-link] [--dry-run] [-v] - sdl sync --all [--copy|--hard-link|--symbolic-link] [--dry-run] [-v] + Usage: sdl sync [...] [--copy|--hard-link|--symbolic-link] [--direct] [--dry-run] [--json] [-v] + sdl sync --all [--copy|--hard-link|--symbolic-link] [--direct] [--dry-run] [--json] [-v] status Show discovered models and sync state. Usage: sdl status [--json] [-v] @@ -343,8 +345,8 @@ def run_help(args: argparse.Namespace, service: StudioLinkService) -> int: doctor Check local StudioLink prerequisites. Usage: sdl doctor [--json] [-v] - prune Remove staging aliases and sync state for models that - Ollama no longer has (reclaims pinned blob space). + prune Safely remove unneeded staging aliases and stale state + after a complete Ollama library scan. Usage: sdl prune [--dry-run] [--json] [-v] upgrade Check for and install a newer StudioLink version. diff --git a/src/studiolink/config.py b/src/studiolink/config.py index aee9135..3b8d138 100644 --- a/src/studiolink/config.py +++ b/src/studiolink/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os import shutil import sys @@ -14,18 +15,42 @@ def _expand_path(env_name: str, default: Path) -> Path: return Path(raw).expanduser() if raw else default.expanduser() +def _default_ollama_models_dir(home: Path) -> Path: + if sys.platform.startswith("linux"): + return Path("/usr/share/ollama/.ollama/models") + return home / ".ollama" / "models" + + def _default_ollama_exe(home: Path) -> Path: if sys.platform == "win32": - return home / "AppData" / "Local" / "Programs" / "Ollama" / "ollama.exe" + return Path( + shutil.which("ollama") + or home / "AppData" / "Local" / "Programs" / "Ollama" / "ollama.exe" + ) return Path(shutil.which("ollama") or "/usr/bin/ollama") def _default_lms_exe(home: Path) -> Path: if sys.platform == "win32": - return home / ".lmstudio" / "bin" / "lms.exe" + return Path(shutil.which("lms") or home / ".lmstudio" / "bin" / "lms.exe") return Path(shutil.which("lms") or home / ".lmstudio" / "bin" / "lms") +def _default_lmstudio_models_dir(home: Path) -> Path: + fallback = home / ".lmstudio" / "models" + settings_path = home / ".lmstudio" / "settings.json" + try: + payload = json.loads(settings_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return fallback + if not isinstance(payload, dict): + return fallback + downloads_folder = payload.get("downloadsFolder") + if not isinstance(downloads_folder, str) or not downloads_folder.strip(): + return fallback + return Path(downloads_folder).expanduser() + + @dataclass(slots=True, frozen=True) class StudioLinkConfig: ollama_exe: Path @@ -46,7 +71,7 @@ def from_env(cls) -> "StudioLinkConfig": # StudioLink-specific override is set. ollama_models_dir = _expand_path( "STUDIOLINK_OLLAMA_MODELS_DIR", - _expand_path("OLLAMA_MODELS", home / ".ollama" / "models"), + _expand_path("OLLAMA_MODELS", _default_ollama_models_dir(home)), ) state_dir = _expand_path("STUDIOLINK_STATE_DIR", home / ".studiolink") return cls( @@ -63,7 +88,7 @@ def from_env(cls) -> "StudioLinkConfig": ollama_blobs_dir=ollama_models_dir / "blobs", lmstudio_models_dir=_expand_path( "STUDIOLINK_LMSTUDIO_MODELS_DIR", - home / ".lmstudio" / "models", + _default_lmstudio_models_dir(home), ), state_dir=state_dir, state_file=state_dir / "state.json", diff --git a/src/studiolink/lmstudio_adapter.py b/src/studiolink/lmstudio_adapter.py index 36480f2..216534f 100644 --- a/src/studiolink/lmstudio_adapter.py +++ b/src/studiolink/lmstudio_adapter.py @@ -1,7 +1,6 @@ from __future__ import annotations import subprocess -from typing import Any from studiolink.config import StudioLinkConfig from studiolink.models import ImportResult, LinkMode @@ -38,9 +37,7 @@ def get_version(self) -> str | None: exc.returncode, exc.stderr, ) from exc - output = "\n".join( - part for part in (result.stdout, result.stderr) if part - ) + output = "\n".join(part for part in (result.stdout, result.stderr) if part) return output.strip() or None def get_import_capabilities(self) -> set[LinkMode]: @@ -123,3 +120,10 @@ def _run( -1, str(exc), ) from exc + except OSError as exc: + raise LMStudioError( + f"Could not execute LM Studio CLI: {exc}", + command, + -1, + str(exc), + ) from exc diff --git a/src/studiolink/models.py b/src/studiolink/models.py index d298bba..f8b5b04 100644 --- a/src/studiolink/models.py +++ b/src/studiolink/models.py @@ -90,6 +90,14 @@ def import_filename(self) -> str: return f"{safe}.gguf" +@dataclass(slots=True, frozen=True) +class OllamaScanReport: + models: tuple[OllamaModel, ...] = () + source_available: bool = True + complete: bool = True + errors: tuple[str, ...] = () + + @dataclass(slots=True, frozen=True) class SyncRecord: canonical_name: str @@ -99,6 +107,7 @@ class SyncRecord: user_repo: str link_mode: LinkMode imported_at: datetime + imported_model_path: Path | None = None import_command: tuple[str, ...] = field(default_factory=tuple) def to_json(self) -> dict[str, object]: @@ -110,11 +119,19 @@ def to_json(self) -> dict[str, object]: "user_repo": self.user_repo, "link_mode": self.link_mode.value, "imported_at": self.imported_at.isoformat(), + "imported_model_path": ( + None + if self.imported_model_path is None + else str(self.imported_model_path) + ), "import_command": list(self.import_command), } @classmethod def from_json(cls, payload: dict[str, object]) -> "SyncRecord": + import_command = payload.get("import_command", []) + if not isinstance(import_command, list): + raise TypeError("import_command must be a list") return cls( canonical_name=str(payload["canonical_name"]), digest=str(payload["digest"]), @@ -123,17 +140,26 @@ def from_json(cls, payload: dict[str, object]) -> "SyncRecord": user_repo=str(payload["user_repo"]), link_mode=LinkMode(str(payload["link_mode"])), imported_at=datetime.fromisoformat(str(payload["imported_at"])), - import_command=tuple( - str(item) for item in payload.get("import_command", []) + imported_model_path=( + Path(str(payload["imported_model_path"])) + if payload.get("imported_model_path") + else None ), + import_command=tuple(str(item) for item in import_command), ) -def is_synced(model: OllamaModel, record: SyncRecord | None) -> bool: - """Check if the sync record matches current model state.""" - if record is None: +def is_synced( + model: OllamaModel, + record: SyncRecord | None, + *, + expected_target: Path | None = None, +) -> bool: + """Check if state and the imported LM Studio file match the model.""" + if record is None or record.digest != model.model_digest: return False - return record.digest == model.model_digest + target = record.imported_model_path or expected_target + return target.is_file() if target is not None else True @dataclass(slots=True, frozen=True) diff --git a/src/studiolink/ollama_adapter.py b/src/studiolink/ollama_adapter.py index 53cc157..906d891 100644 --- a/src/studiolink/ollama_adapter.py +++ b/src/studiolink/ollama_adapter.py @@ -2,14 +2,15 @@ import json import logging +import re from pathlib import Path from studiolink.config import StudioLinkConfig -from studiolink.models import OllamaModel - +from studiolink.models import OllamaModel, OllamaScanReport MODEL_MEDIA_TYPE = "application/vnd.ollama.image.model" GGUF_MAGIC = b"GGUF" +SHA256_DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-fA-F]{64}$") logger = logging.getLogger("studiolink") @@ -19,19 +20,44 @@ def __init__(self, config: StudioLinkConfig) -> None: self.config = config def scan_models(self) -> list[OllamaModel]: + return list(self.scan_report().models) + + def scan_report(self) -> OllamaScanReport: manifests_dir = self.config.ollama_manifests_dir logger.debug("Scanning manifests directory: %s", manifests_dir) - if not manifests_dir.exists(): - logger.warning("Manifests directory does not exist: %s", manifests_dir) - return [] + if not manifests_dir.is_dir(): + message = f"Manifests directory does not exist: {manifests_dir}" + logger.warning(message) + return OllamaScanReport( + source_available=False, + complete=False, + errors=(message,), + ) models: list[OllamaModel] = [] - manifest_files = sorted( - path for path in manifests_dir.rglob("*") if path.is_file() - ) + errors: list[str] = [] + try: + manifest_files = sorted( + path for path in manifests_dir.rglob("*") if path.is_file() + ) + except OSError as exc: + message = f"Could not enumerate Ollama manifests: {exc}" + logger.warning(message) + return OllamaScanReport( + source_available=False, + complete=False, + errors=(message,), + ) + logger.debug("Found %d manifest file(s)", len(manifest_files)) for manifest_path in manifest_files: - model = self._parse_manifest(manifest_path) + try: + model = self._parse_manifest(manifest_path) + except (OSError, UnicodeError) as exc: + message = f"Could not inspect manifest {manifest_path}: {exc}" + logger.warning(message) + errors.append(message) + continue if model is not None: models.append(model) logger.debug( @@ -41,7 +67,12 @@ def scan_models(self) -> list[OllamaModel]: ) else: logger.debug("Skipped manifest: %s", manifest_path) - return models + return OllamaScanReport( + models=tuple(models), + source_available=True, + complete=not errors, + errors=tuple(errors), + ) def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: logger.debug("Parsing manifest: %s", manifest_path) @@ -83,8 +114,13 @@ def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: issues.append("manifest is not a JSON object") manifest = {} + layers = manifest.get("layers", []) + if not isinstance(layers, list): + issues.append("manifest layers must be a list") + layers = [] + model_layer = None - for layer in manifest.get("layers", []): + for layer in layers: if isinstance(layer, dict) and layer.get("mediaType") == MODEL_MEDIA_TYPE: model_layer = layer break @@ -102,6 +138,8 @@ def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: declared_size = self._as_int(model_layer.get("size")) if model_digest is None: issues.append("model layer is missing a digest") + elif SHA256_DIGEST_PATTERN.fullmatch(model_digest) is None: + issues.append("model layer has an invalid SHA-256 digest") else: blob_path = self.config.ollama_blobs_dir / model_digest.replace( ":", "-" @@ -140,8 +178,10 @@ def _parse_manifest(self, manifest_path: Path) -> OllamaModel | None: @staticmethod def _as_int(value: object) -> int | None: + if not isinstance(value, (str, int, float)): + return None try: - return int(value) if value is not None else None + return int(value) except (TypeError, ValueError): return None diff --git a/src/studiolink/ports.py b/src/studiolink/ports.py index e90dd64..1704185 100644 --- a/src/studiolink/ports.py +++ b/src/studiolink/ports.py @@ -1,14 +1,23 @@ from __future__ import annotations + from typing import Protocol, runtime_checkable -from studiolink.config import StudioLinkConfig -from studiolink.models import ImportResult, LinkMode, OllamaModel + +from studiolink.models import ImportResult, LinkMode, OllamaModel, OllamaScanReport + + @runtime_checkable class OllamaPort(Protocol): def scan_models(self) -> list[OllamaModel]: ... + + def scan_report(self) -> OllamaScanReport: ... + + @runtime_checkable class LMStudioPort(Protocol): def get_version(self) -> str | None: ... + def get_import_capabilities(self) -> set[LinkMode]: ... + def import_model( self, source_path: str, @@ -16,4 +25,4 @@ def import_model( user_repo: str, link_mode: LinkMode, dry_run: bool = False, - ) -> ImportResult: ... \ No newline at end of file + ) -> ImportResult: ... diff --git a/src/studiolink/service.py b/src/studiolink/service.py index 20a4351..55fd20d 100644 --- a/src/studiolink/service.py +++ b/src/studiolink/service.py @@ -5,21 +5,22 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from studiolink.ollama_adapter import OllamaAdapter -from studiolink.lmstudio_adapter import LMStudioAdapter from studiolink.config import StudioLinkConfig +from studiolink.lmstudio_adapter import LMStudioAdapter from studiolink.models import ( DoctorCheck, ImportMode, - is_synced, LinkMode, ModelReadiness, OllamaModel, + OllamaScanReport, PruneReport, PruneResult, SyncRecord, SyncResult, + is_synced, ) +from studiolink.ollama_adapter import OllamaAdapter from studiolink.state import StateStore from studiolink.syncer import Syncer, same_file @@ -59,12 +60,16 @@ def __init__( self.state = StateStore(self.config.state_file) def scan(self) -> list[OllamaModel]: + report = self.scan_report() + return list(report.models) + + def scan_report(self) -> OllamaScanReport: logger.debug( "Scanning Ollama manifests at %s", self.config.ollama_manifests_dir ) - models = self.ollama.scan_models() - logger.debug("Discovered %d model(s)", len(models)) - return models + report = self.ollama.scan_report() + logger.debug("Discovered %d model(s)", len(report.models)) + return report def status(self) -> list[StatusEntry]: models = self.scan() @@ -72,7 +77,7 @@ def status(self) -> list[StatusEntry]: return [ StatusEntry( model=model, - synced=is_synced(model, records.get(model.canonical_name)), + synced=self._record_is_synced(model, records.get(model.canonical_name)), sync_record=records.get(model.canonical_name), ) for model in models @@ -111,17 +116,36 @@ def sync( return syncer.sync(selected, mode, import_mode, dry_run) def prune(self, *, dry_run: bool = False) -> PruneReport: - """Remove staging aliases and sync records for models Ollama no longer has. + """Safely remove unneeded aliases and stale sync records. - Hard-linked aliases keep deleted Ollama blobs alive on disk; this is - the only way to reclaim that space. + Destructive work requires a complete Ollama scan, and aliases needed + by symbolic LM Studio imports are preserved. """ - models = self.scan() + scan = self.scan_report() + if not scan.source_available: + details = "; ".join(scan.errors) or "source is unavailable" + raise RuntimeError(f"Ollama library could not be scanned: {details}") + if not scan.complete: + details = "; ".join(scan.errors) or "unknown scan error" + raise RuntimeError(f"Ollama manifest scan was incomplete: {details}") + + models = list(scan.models) expected: dict[str, OllamaModel] = { model.import_filename: model for model in models if model.readiness is ModelReadiness.READY and model.blob_path is not None } + records = self.state.get_all_records() + unready_names = { + model.canonical_name + for model in models + if model.readiness is not ModelReadiness.READY + } + protected_aliases = { + record.import_alias_path + for name, record in records.items() + if record.link_mode is LinkMode.SYMBOLIC_LINK or name in unready_names + } alias_results: list[PruneResult] = [] staging = self.config.import_staging_dir @@ -129,6 +153,9 @@ def prune(self, *, dry_run: bool = False) -> PruneReport: for path in sorted(staging.iterdir()): if not path.is_file(): continue + if path in protected_aliases: + logger.debug("Preserving required import alias: %s", path) + continue model = expected.get(path.name) if model is not None and same_file(path, model.blob_path or path): continue @@ -159,20 +186,20 @@ def prune(self, *, dry_run: bool = False) -> PruneReport: ) ) - # Only prune records when Ollama's library is visible; an empty scan - # usually means the models dir is misconfigured, not that everything - # was removed. - records_removed: tuple[str, ...] = () - if models: - known = {model.canonical_name for model in models} - records = self.state.get_all_records() - stale_names = sorted( - name for name in records if name not in known - ) - if stale_names and not dry_run: - for name in stale_names: - self.state.remove(name) - records_removed = tuple(stale_names) + known = {model.canonical_name for model in models} + stale_names = sorted( + name + for name, record in records.items() + if name not in known and record.link_mode is not LinkMode.SYMBOLIC_LINK + ) + if stale_names and not dry_run: + remaining = { + name: record + for name, record in records.items() + if name not in stale_names + } + self.state.save(remaining) + records_removed = tuple(stale_names) logger.debug( "Prune complete: %d alias(es), %d record(s) (dry_run=%s)", @@ -242,9 +269,7 @@ def doctor(self) -> list[DoctorCheck]: ) except Exception as exc: logger.debug("Doctor: LM Studio capabilities check failed: %s", exc) - checks.append( - DoctorCheck("lm studio import capabilities", False, str(exc)) - ) + checks.append(DoctorCheck("lm studio import capabilities", False, str(exc))) models = self.scan() checks.append( @@ -255,9 +280,7 @@ def doctor(self) -> list[DoctorCheck]: ) ) - stale_models = [ - m for m in models if m.readiness is ModelReadiness.STALE - ] + stale_models = [m for m in models if m.readiness is ModelReadiness.STALE] checks.append( DoctorCheck( "ollama blob presence", @@ -277,13 +300,23 @@ def doctor(self) -> list[DoctorCheck]: ) ) - checks.append( - DoctorCheck( - "import alias directory", - self.config.import_staging_dir.exists(), - str(self.config.import_staging_dir), + staging = self.config.import_staging_dir + if staging.exists(): + alias_check = DoctorCheck( + "import alias directory", staging.is_dir(), str(staging) ) - ) + else: + parent = staging.parent + while not parent.exists() and parent != parent.parent: + parent = parent.parent + creatable = parent.is_dir() and os.access(parent, os.W_OK) + details = ( + f"{staging} (will be created)" + if creatable + else f"{staging} (parent is not writable)" + ) + alias_check = DoctorCheck("import alias directory", creatable, details) + checks.append(alias_check) return checks @@ -314,6 +347,16 @@ def _volume_compatibility_check(self) -> DoctorCheck: ) return DoctorCheck("hard-link volume compatibility", ok, details) + def _record_is_synced(self, model: OllamaModel, record: SyncRecord | None) -> bool: + expected_target = None + if record is not None and record.imported_model_path is None: + expected_target = ( + self.config.lmstudio_models_dir + / record.user_repo + / record.import_alias_path.name + ) + return is_synced(model, record, expected_target=expected_target) + def _select_models( self, discovered: list[OllamaModel], requested_names: list[str] ) -> list[OllamaModel]: @@ -335,9 +378,7 @@ def _select_models( matches = lookup.get(name, []) if not matches: available = ", ".join(sorted(lookup.keys())) - raise ValueError( - f"model not found: {name}. Available: {available}" - ) + raise ValueError(f"model not found: {name}. Available: {available}") selected.extend(matches) return selected diff --git a/src/studiolink/state.py b/src/studiolink/state.py index b822436..6a91922 100644 --- a/src/studiolink/state.py +++ b/src/studiolink/state.py @@ -3,6 +3,7 @@ import json import logging import os +from collections.abc import Mapping from pathlib import Path from studiolink.models import SyncRecord @@ -59,12 +60,10 @@ def _parse_record(name: str, record_data: object) -> SyncRecord | None: logger.warning("Skipping corrupt sync record %r: %s", name, exc) return None - def _save_raw(self, records: dict[str, object]) -> None: + def _save_raw(self, records: Mapping[str, object]) -> None: """Persist raw JSON dict to disk atomically.""" self.path.parent.mkdir(parents=True, exist_ok=True) - payload = json.dumps( - {"schema_version": 1, "sync_records": records}, indent=2 - ) + payload = json.dumps({"schema_version": 1, "sync_records": records}, indent=2) tmp_path = self.path.parent / (self.path.name + ".tmp") tmp_path.write_text(payload, encoding="utf-8") os.replace(tmp_path, self.path) @@ -82,8 +81,7 @@ def _load_raw(self) -> dict[str, object]: return {} if not isinstance(payload, dict): logger.warning( - "Corrupted state file at %s: expected a JSON object. " - "Starting fresh.", + "Corrupted state file at %s: expected a JSON object. Starting fresh.", self.path, ) return {} diff --git a/src/studiolink/syncer.py b/src/studiolink/syncer.py index 3ac18e3..4e16a93 100644 --- a/src/studiolink/syncer.py +++ b/src/studiolink/syncer.py @@ -11,12 +11,12 @@ from studiolink.models import ( ImportMode, ImportResult, - is_synced, LinkMode, ModelReadiness, OllamaModel, SyncRecord, SyncResult, + is_synced, ) from studiolink.ports import LMStudioPort from studiolink.state import StateStore @@ -30,7 +30,10 @@ def same_file(left: Path, right: Path) -> bool: left_stat, right_stat = left.stat(), right.stat() except OSError: return False - return (left_stat.st_dev, left_stat.st_ino) == (right_stat.st_dev, right_stat.st_ino) + return (left_stat.st_dev, left_stat.st_ino) == ( + right_stat.st_dev, + right_stat.st_ino, + ) class Syncer: @@ -84,7 +87,14 @@ def _sync_one( model.readiness, ) existing = records.get(model.canonical_name) - if is_synced(model, existing): + expected_target = None + if existing is not None and existing.imported_model_path is None: + expected_target = ( + self.config.lmstudio_models_dir + / existing.user_repo + / existing.import_alias_path.name + ) + if is_synced(model, existing, expected_target=expected_target): logger.debug("Model %s is already synced, skipping", model.canonical_name) return SyncResult( model=model, @@ -120,7 +130,7 @@ def _sync_one( logger.debug("Using Ollama blob directly: %s", alias_path) else: try: - alias_path, alias_created = self._ensure_import_alias(model) + alias_path, alias_created = self._ensure_import_alias(model, link_mode) logger.debug( "Importing model via LM Studio: %s (mode=%s, dry_run=%s)", alias_path, @@ -136,6 +146,7 @@ def _sync_one( ) import_error: Exception | None = None + import_result: ImportResult | None = None try: import_result = self.lmstudio.import_model( str(alias_path), @@ -149,17 +160,30 @@ def _sync_one( logger.debug("LM Studio import failed: %s", exc) import_error = exc finally: + should_remove_alias = dry_run or link_mode is LinkMode.COPY if ( - dry_run - and alias_created + should_remove_alias + and (alias_created or link_mode is LinkMode.COPY) and alias_path.exists() and import_mode is not ImportMode.DIRECT ): - logger.debug("Cleaning up dry-run alias: %s", alias_path) - alias_path.unlink() + logger.debug("Cleaning up temporary import alias: %s", alias_path) + try: + alias_path.unlink() + alias_path.parent.rmdir() + except OSError as exc: + logger.warning( + "Could not clean up import alias %s: %s", alias_path, exc + ) if import_error is not None: return SyncResult(model=model, status="error", message=str(import_error)) + if import_result is None: + return SyncResult( + model=model, + status="error", + message="LM Studio import returned no result", + ) record = SyncRecord( canonical_name=model.canonical_name, @@ -169,6 +193,9 @@ def _sync_one( user_repo=model.user_repo, link_mode=link_mode, imported_at=datetime.now(tz=timezone.utc), + imported_model_path=( + self.config.lmstudio_models_dir / model.user_repo / alias_path.name + ), import_command=import_result.command, ) return SyncResult( @@ -178,13 +205,19 @@ def _sync_one( record=record, ) - def _ensure_import_alias(self, model: OllamaModel) -> tuple[Path, bool]: - return _ensure_import_alias(model, self.config.import_staging_dir) + def _ensure_import_alias( + self, model: OllamaModel, link_mode: LinkMode + ) -> tuple[Path, bool]: + staging_dir = self.config.import_staging_dir + if link_mode is LinkMode.COPY: + # The alias controls the human-readable LM Studio filename. Keep + # this temporary hard link on the Ollama filesystem so copy mode + # still works when StudioLink state lives on another volume. + staging_dir = self.config.ollama_models_dir / ".studiolink-imports" + return _ensure_import_alias(model, staging_dir) -def _ensure_import_alias( - model: OllamaModel, staging_dir: Path -) -> tuple[Path, bool]: +def _ensure_import_alias(model: OllamaModel, staging_dir: Path) -> tuple[Path, bool]: alias_path = staging_dir / model.import_filename if model.blob_path is None: raise RuntimeError(f"No blob path for model {model.canonical_name}") @@ -202,9 +235,7 @@ def _ensure_import_alias( logger.debug("Created hard link: %s -> %s", alias_path, model.blob_path) return alias_path, True except OSError as exc: - raise RuntimeError( - f"Failed to create import alias (hard link): {exc}" - ) from exc + raise RuntimeError(f"Failed to create import alias (hard link): {exc}") from exc def _format_import_message(result: ImportResult) -> str: diff --git a/tests/conftest.py b/tests/conftest.py index a1d2549..11c5417 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,9 +14,14 @@ if str(_SRC) not in sys.path: sys.path.insert(0, str(_SRC)) -from studiolink.config import StudioLinkConfig -from studiolink.lmstudio_adapter import LMStudioError -from studiolink.models import ImportResult, LinkMode, OllamaModel, SyncRecord +from studiolink.config import StudioLinkConfig # noqa: E402 +from studiolink.lmstudio_adapter import LMStudioError # noqa: E402 +from studiolink.models import ( # noqa: E402 + ImportResult, + LinkMode, + OllamaModel, + SyncRecord, +) MODEL_MEDIA_TYPE = "application/vnd.ollama.image.model" DIGEST_A = "sha256:" + "a" * 64 @@ -95,9 +100,7 @@ def write_manifest( if content is None: content = { "schemaVersion": 2, - "layers": [ - {"mediaType": MODEL_MEDIA_TYPE, "digest": digest, "size": size} - ], + "layers": [{"mediaType": MODEL_MEDIA_TYPE, "digest": digest, "size": size}], } manifest_path.write_text(json.dumps(content), encoding="utf-8") return manifest_path @@ -154,6 +157,7 @@ def __init__(self) -> None: self.calls: list[dict] = [] self.fail_for: set[str] = set() self.timeout_for: set[str] = set() + self.models_dir: Path | None = None def get_version(self) -> str | None: return "0.3.17 (fake)" @@ -186,6 +190,10 @@ def import_model( ) if source_path in self.timeout_for: raise subprocess.TimeoutExpired(cmd=["lms"], timeout=3600) + if not dry_run and self.models_dir is not None: + target = self.models_dir / user_repo / Path(source_path).name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(Path(source_path).read_bytes()) return ImportResult( command=("lms", "import", source_path), stdout="", @@ -207,6 +215,7 @@ def make_syncer(make_config, fake_lmstudio): def _make(config: StudioLinkConfig | None = None): cfg = config or make_config() + fake_lmstudio.models_dir = cfg.lmstudio_models_dir syncer = Syncer(cfg, fake_lmstudio, StateStore(cfg.state_file)) return syncer, cfg, fake_lmstudio diff --git a/tests/test_cli.py b/tests/test_cli.py index f0f15fa..6198958 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -31,8 +31,15 @@ def status(self): self.calls.append(("status",)) return [StatusEntry(model=self._model, synced=False, sync_record=None)] - def sync(self, *, model_names=None, sync_all=False, link_mode=None, - import_mode=ImportMode.ALIAS, dry_run=False): + def sync( + self, + *, + model_names=None, + sync_all=False, + link_mode=None, + import_mode=ImportMode.ALIAS, + dry_run=False, + ): self.calls.append( ("sync", model_names, sync_all, link_mode, import_mode, dry_run) ) @@ -100,7 +107,8 @@ def test_verbose_flag_after_subcommand(self, fake_service, caplog): caplog.set_level(logging.DEBUG, logger="studiolink") assert cli.main(["scan", "-v"]) == 0 debug_records = [ - r for r in caplog.records + r + for r in caplog.records if r.name == "studiolink" and r.levelno == logging.DEBUG ] assert debug_records, "expected debug logging to be enabled" @@ -109,7 +117,8 @@ def test_verbose_flag_before_subcommand(self, fake_service, caplog): caplog.set_level(logging.DEBUG, logger="studiolink") assert cli.main(["-v", "scan"]) == 0 debug_records = [ - r for r in caplog.records + r + for r in caplog.records if r.name == "studiolink" and r.levelno == logging.DEBUG ] assert debug_records, "expected debug logging to be enabled" @@ -118,7 +127,8 @@ def test_no_verbose_means_no_debug(self, fake_service, caplog): caplog.set_level(logging.DEBUG, logger="studiolink") assert cli.main(["scan"]) == 0 debug_records = [ - r for r in caplog.records + r + for r in caplog.records if r.name == "studiolink" and r.levelno == logging.DEBUG ] assert not debug_records @@ -162,7 +172,9 @@ def test_sync_json_output(self, fake_service, capsys): def test_sync_error_result_exits_nonzero(self, fake_service, monkeypatch, capsys): def failing_sync(**kwargs): - return [SyncResult(model=fake_service._model, status="error", message="bad")] + return [ + SyncResult(model=fake_service._model, status="error", message="bad") + ] monkeypatch.setattr(fake_service, "sync", failing_sync) assert cli.main(["sync", "llama3:1b"]) == 1 @@ -217,9 +229,7 @@ def test_prune_reports_aliases(self, fake_service, monkeypatch, capsys): report = PruneReport( dry_run=False, - aliases=( - PruneResult(Path("old.gguf"), 1024, True, "model gone", True), - ), + aliases=(PruneResult(Path("old.gguf"), 1024, True, "model gone", True),), records_removed=("old:model",), ) monkeypatch.setattr(fake_service, "prune", lambda *, dry_run=False: report) diff --git a/tests/test_config.py b/tests/test_config.py index 8017f85..66513b1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import json import sys from pathlib import Path @@ -44,24 +45,79 @@ def test_studiolink_var_wins_over_ollama_var(self, monkeypatch, tmp_path): assert config.ollama_models_dir == tmp_path / "studio" +class TestLMStudioModelsDirectory: + def test_uses_downloads_folder_from_lmstudio_settings(self, monkeypatch, tmp_path): + custom = tmp_path / "large-drive" / "lm-models" + settings = tmp_path / ".lmstudio" / "settings.json" + settings.parent.mkdir(parents=True) + settings.write_text( + json.dumps({"downloadsFolder": str(custom)}), encoding="utf-8" + ) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + + config = StudioLinkConfig.from_env() + + assert config.lmstudio_models_dir == custom + + def test_explicit_override_wins_over_lmstudio_settings(self, monkeypatch, tmp_path): + configured = tmp_path / "settings-models" + override = tmp_path / "override-models" + settings = tmp_path / ".lmstudio" / "settings.json" + settings.parent.mkdir(parents=True) + settings.write_text( + json.dumps({"downloadsFolder": str(configured)}), encoding="utf-8" + ) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + monkeypatch.setenv("STUDIOLINK_LMSTUDIO_MODELS_DIR", str(override)) + + config = StudioLinkConfig.from_env() + + assert config.lmstudio_models_dir == override + + class TestPlatformDefaults: def test_windows_defaults(self, monkeypatch, tmp_path): monkeypatch.setattr(sys, "platform", "win32") - monkeypatch.setattr( - Path, "home", classmethod(lambda cls: tmp_path) - ) + monkeypatch.setattr(config_module.shutil, "which", lambda name: None) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) config = StudioLinkConfig.from_env() assert config.ollama_exe == ( tmp_path / "AppData" / "Local" / "Programs" / "Ollama" / "ollama.exe" ) assert config.lms_exe == tmp_path / ".lmstudio" / "bin" / "lms.exe" + def test_windows_prefers_ollama_on_path(self, monkeypatch, tmp_path): + found = tmp_path / "tools" / "ollama.exe" + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr( + config_module.shutil, + "which", + lambda name: str(found) if name == "ollama" else None, + ) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + + config = StudioLinkConfig.from_env() + + assert config.ollama_exe == found + + def test_windows_prefers_lms_on_path(self, monkeypatch, tmp_path): + found = tmp_path / "tools" / "lms.exe" + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr( + config_module.shutil, + "which", + lambda name: str(found) if name == "lms" else None, + ) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + + config = StudioLinkConfig.from_env() + + assert config.lms_exe == found + def test_posix_defaults_without_path_lookup(self, monkeypatch, tmp_path): monkeypatch.setattr(sys, "platform", "linux") monkeypatch.setattr(config_module.shutil, "which", lambda name: None) - monkeypatch.setattr( - Path, "home", classmethod(lambda cls: tmp_path) - ) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) config = StudioLinkConfig.from_env() assert config.ollama_exe == Path("/usr/bin/ollama") assert config.lms_exe == tmp_path / ".lmstudio" / "bin" / "lms" @@ -73,19 +129,25 @@ def test_posix_prefers_executables_on_path(self, monkeypatch, tmp_path): } monkeypatch.setattr(sys, "platform", "linux") monkeypatch.setattr(config_module.shutil, "which", lambda name: found.get(name)) - monkeypatch.setattr( - Path, "home", classmethod(lambda cls: tmp_path) - ) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) config = StudioLinkConfig.from_env() assert config.ollama_exe == Path("/usr/local/bin/ollama") assert config.lms_exe == Path("/home/user/.local/bin/lms") - def test_posix_default_models_dirs(self, monkeypatch, tmp_path): + def test_linux_default_models_dirs(self, monkeypatch, tmp_path): monkeypatch.setattr(sys, "platform", "linux") - monkeypatch.setattr( - Path, "home", classmethod(lambda cls: tmp_path) - ) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + config = StudioLinkConfig.from_env() - assert config.ollama_models_dir == tmp_path / ".ollama" / "models" + + assert config.ollama_models_dir == Path("/usr/share/ollama/.ollama/models") assert config.lmstudio_models_dir == tmp_path / ".lmstudio" / "models" assert config.state_dir == tmp_path / ".studiolink" + + def test_macos_default_models_dir(self, monkeypatch, tmp_path): + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + + config = StudioLinkConfig.from_env() + + assert config.ollama_models_dir == tmp_path / ".ollama" / "models" diff --git a/tests/test_lmstudio_adapter.py b/tests/test_lmstudio_adapter.py new file mode 100644 index 0000000..8b9858c --- /dev/null +++ b/tests/test_lmstudio_adapter.py @@ -0,0 +1,23 @@ +import pytest + +from studiolink.lmstudio_adapter import LMStudioAdapter, LMStudioError +from studiolink.models import LinkMode + + +def test_import_wraps_os_execution_error(make_config, monkeypatch): + config = make_config() + adapter = LMStudioAdapter(config) + + def missing_executable(*args, **kwargs): + raise FileNotFoundError("lms is missing") + + monkeypatch.setattr( + "studiolink.lmstudio_adapter.subprocess.run", missing_executable + ) + + with pytest.raises(LMStudioError, match="Could not execute LM Studio CLI"): + adapter.import_model( + "model.gguf", + user_repo="ollama/model", + link_mode=LinkMode.COPY, + ) diff --git a/tests/test_models.py b/tests/test_models.py index 90cd364..fe9b8f1 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -52,9 +52,7 @@ def test_library_repository(self): assert make_ollama_model("llama3:1b").user_repo == "ollama/llama3" def test_namespaced_repository(self): - model = make_ollama_model( - "gemma:2b", namespace="marella", repository="gemma" - ) + model = make_ollama_model("gemma:2b", namespace="marella", repository="gemma") assert model.user_repo == "ollama/marella-gemma" def test_custom_registry_dots_replaced(self): @@ -136,9 +134,7 @@ def test_prune_report_json(self): report = PruneReport( dry_run=True, - aliases=( - PruneResult(Path("a"), 1, True, "r", False), - ), + aliases=(PruneResult(Path("a"), 1, True, "r", False),), records_removed=("old:model",), ) payload = report.to_json() diff --git a/tests/test_ollama_adapter.py b/tests/test_ollama_adapter.py index b37d100..f8a4eb7 100644 --- a/tests/test_ollama_adapter.py +++ b/tests/test_ollama_adapter.py @@ -1,3 +1,5 @@ +from pathlib import Path + from conftest import ( DIGEST_A, blob_filename, @@ -76,6 +78,29 @@ def test_missing_model_layer(self, make_config): class TestMalformedManifests: + def test_io_failure_is_isolated_and_marks_scan_incomplete( + self, make_config, monkeypatch + ): + config = make_config() + broken = write_manifest(config, repository="broken", digest=DIGEST_A) + write_manifest(config, repository="healthy", digest=DIGEST_A) + write_blob(config, DIGEST_A) + real_read_text = Path.read_text + + def flaky_read_text(path, *args, **kwargs): + if path == broken: + raise OSError("access denied") + return real_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", flaky_read_text) + + report = OllamaAdapter(config).scan_report() + + assert report.source_available is True + assert report.complete is False + assert [model.repository for model in report.models] == ["healthy"] + assert any("access denied" in error for error in report.errors) + def test_non_dict_json_is_invalid_not_crash(self, make_config): # Regression: a stray JSON array under manifests/ used to raise # AttributeError and kill the whole scan. @@ -92,6 +117,25 @@ def test_invalid_json_is_invalid_not_crash(self, make_config): assert model.readiness is ModelReadiness.INVALID assert any("invalid manifest JSON" in issue for issue in model.issues) + def test_non_list_layers_is_invalid_not_crash(self, make_config): + config = make_config() + write_manifest(config, content={"layers": None}) + + model = scan_one(config) + + assert model.readiness is ModelReadiness.INVALID + assert "manifest layers must be a list" in model.issues + + def test_invalid_digest_is_rejected_before_path_resolution(self, make_config): + config = make_config() + write_manifest(config, digest="../../outside") + + model = scan_one(config) + + assert model.readiness is ModelReadiness.INVALID + assert model.blob_path is None + assert "model layer has an invalid SHA-256 digest" in model.issues + def test_layer_without_digest(self, make_config): config = make_config() write_manifest( diff --git a/tests/test_service.py b/tests/test_service.py index 6ac9579..bc49bd7 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,22 +1,27 @@ import os import types +from dataclasses import replace +from pathlib import Path +import pytest from conftest import ( DIGEST_A, - make_config, make_sync_record, write_blob, write_manifest, ) import studiolink.service as service_module +from studiolink.models import LinkMode from studiolink.ollama_adapter import OllamaAdapter from studiolink.service import StudioLinkService from studiolink.state import StateStore def build_service(config, lmstudio): - return StudioLinkService(config=config, ollama=OllamaAdapter(config), lmstudio=lmstudio) + return StudioLinkService( + config=config, ollama=OllamaAdapter(config), lmstudio=lmstudio + ) class TestDoctorVolumeCheck: @@ -49,7 +54,7 @@ def test_mismatched_devices_fail(self, make_config, fake_lmstudio, monkeypatch): devices = {} def fake_stat(path, *args, **kwargs): - stat = real_stat(path, *args, **kwargs) + real_stat(path, *args, **kwargs) key = str(path) if key not in devices: devices[key] = len(devices) + 1 @@ -66,6 +71,21 @@ def test_skips_when_directories_missing(self, make_config, fake_lmstudio): assert check.ok is True assert "skipped" in check.details + def test_doctor_accepts_import_directory_that_has_not_been_created_yet( + self, make_config, fake_lmstudio + ): + config = make_config() + config.ollama_manifests_dir.mkdir(parents=True) + config.ollama_blobs_dir.mkdir(parents=True) + config.lmstudio_models_dir.mkdir(parents=True) + service = build_service(config, fake_lmstudio) + + checks = service.doctor() + + aliases = next(c for c in checks if c.name == "import alias directory") + assert aliases.ok is True + assert "will be created" in aliases.details + def test_doctor_includes_volume_check(self, make_config, fake_lmstudio): config = make_config() for directory in ( @@ -100,6 +120,81 @@ def test_removes_orphaned_alias(self, make_config, fake_lmstudio): assert report.aliases[0].would_free is True # last link assert report.aliases[0].size == 104 + def test_refuses_prune_after_partial_manifest_scan( + self, make_config, fake_lmstudio, monkeypatch + ): + config = make_config() + service = build_service(config, fake_lmstudio) + broken = write_manifest(config, repository="broken", digest=DIGEST_A) + write_manifest(config, repository="healthy", digest=DIGEST_A) + write_blob(config, DIGEST_A) + config.import_staging_dir.mkdir(parents=True) + alias = config.import_staging_dir / "orphan.gguf" + alias.write_bytes(b"GGUFxxxx") + real_read_text = Path.read_text + + def flaky_read_text(path, *args, **kwargs): + if path == broken: + raise OSError("access denied") + return real_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", flaky_read_text) + + with pytest.raises(RuntimeError, match="scan was incomplete"): + service.prune() + + assert alias.exists() + + def test_preserves_alias_when_current_manifest_is_not_ready( + self, make_config, fake_lmstudio + ): + config = make_config() + service = build_service(config, fake_lmstudio) + write_manifest(config, digest=DIGEST_A) + blob = write_blob(config, DIGEST_A) + model = OllamaAdapter(config).scan_models()[0] + config.import_staging_dir.mkdir(parents=True) + alias = config.import_staging_dir / model.import_filename + os.link(blob, alias) + StateStore(config.state_file).upsert( + replace( + make_sync_record(model.canonical_name, DIGEST_A), + import_alias_path=alias, + ) + ) + blob.unlink() + + report = service.prune() + + assert report.aliases == () + assert alias.exists() + + def test_preserves_alias_required_by_symbolic_import( + self, make_config, fake_lmstudio + ): + config = make_config() + service = build_service(config, fake_lmstudio) + manifest = write_manifest(config, digest=DIGEST_A) + blob = write_blob(config, DIGEST_A) + model = OllamaAdapter(config).scan_models()[0] + + config.import_staging_dir.mkdir(parents=True) + alias = config.import_staging_dir / model.import_filename + os.link(blob, alias) + record = replace( + make_sync_record(model.canonical_name, DIGEST_A), + import_alias_path=alias, + link_mode=LinkMode.SYMBOLIC_LINK, + ) + StateStore(config.state_file).upsert(record) + manifest.unlink() + blob.unlink() + + report = service.prune() + + assert report.aliases == () + assert alias.exists() + def test_keeps_healthy_alias(self, make_config, fake_lmstudio): config = make_config() service = build_service(config, fake_lmstudio) @@ -115,7 +210,9 @@ def test_keeps_healthy_alias(self, make_config, fake_lmstudio): assert report.aliases == () assert (config.import_staging_dir / model.import_filename).exists() - def test_removes_same_name_alias_pointing_elsewhere(self, make_config, fake_lmstudio): + def test_removes_same_name_alias_pointing_elsewhere( + self, make_config, fake_lmstudio + ): config = make_config() service = build_service(config, fake_lmstudio) write_manifest(config, digest=DIGEST_A) @@ -164,19 +261,39 @@ def test_removes_stale_records_keeps_current(self, make_config, fake_lmstudio): records = store.get_all_records() assert set(records) == {"llama3:1b"} - def test_keeps_records_when_nothing_discovered(self, make_config, fake_lmstudio): - # An empty scan usually means a misconfigured models dir, not that - # every model was removed - records must survive. + def test_refuses_prune_when_ollama_library_is_unavailable( + self, make_config, fake_lmstudio + ): + # A missing models directory is not proof that the user removed every + # model. Prune must not delete either aliases or state in that case. config = make_config() service = build_service(config, fake_lmstudio) store = StateStore(config.state_file) store.upsert(make_sync_record("llama3:1b", DIGEST_A)) + config.import_staging_dir.mkdir(parents=True) + alias = config.import_staging_dir / "llama3-1b-old.gguf" + alias.write_bytes(b"GGUFxxxx") - report = service.prune() + with pytest.raises(RuntimeError, match="Ollama library could not be scanned"): + service.prune() - assert report.records_removed == () + assert alias.exists() assert set(store.get_all_records()) == {"llama3:1b"} + def test_empty_but_available_library_removes_non_symbolic_records( + self, make_config, fake_lmstudio + ): + config = make_config() + config.ollama_manifests_dir.mkdir(parents=True) + service = build_service(config, fake_lmstudio) + store = StateStore(config.state_file) + store.upsert(make_sync_record("removed:model", DIGEST_A)) + + report = service.prune() + + assert report.records_removed == ("removed:model",) + assert store.get_all_records() == {} + def test_reports_space_freed_for_last_link(self, make_config, fake_lmstudio): # Simulate `ollama rm`: manifest and blob deleted, but the hard-linked # alias keeps the bytes on disk. Prune reclaims them. @@ -212,13 +329,22 @@ def test_status_reports_pending(self, make_config, fake_lmstudio): assert entries[0].display_status == "pending" assert entries[0].synced is False - def test_status_reports_synced(self, make_config, fake_lmstudio): + def test_status_reports_synced_only_when_imported_target_exists( + self, make_config, fake_lmstudio + ): config = make_config() service = build_service(config, fake_lmstudio) write_manifest(config, digest=DIGEST_A) write_blob(config, DIGEST_A) + model = OllamaAdapter(config).scan_models()[0] + target = config.lmstudio_models_dir / model.user_repo / model.import_filename + target.parent.mkdir(parents=True) + target.write_bytes(b"GGUFxxxx") StateStore(config.state_file).upsert( - make_sync_record("llama3:1b", DIGEST_A) + replace( + make_sync_record("llama3:1b", DIGEST_A), + imported_model_path=target, + ) ) entries = service.status() @@ -226,6 +352,54 @@ def test_status_reports_synced(self, make_config, fake_lmstudio): assert entries[0].display_status == "synced" assert entries[0].synced is True + def test_status_reconciles_legacy_record_with_derived_target( + self, make_config, fake_lmstudio + ): + config = make_config() + service = build_service(config, fake_lmstudio) + write_manifest(config, digest=DIGEST_A) + write_blob(config, DIGEST_A) + record = make_sync_record("llama3:1b", DIGEST_A) + target = ( + config.lmstudio_models_dir + / record.user_repo + / record.import_alias_path.name + ) + target.parent.mkdir(parents=True) + target.write_bytes(b"GGUFxxxx") + StateStore(config.state_file).upsert(record) + + entries = service.status() + + assert entries[0].display_status == "synced" + assert entries[0].synced is True + + target.unlink() + entries = service.status() + + assert entries[0].display_status == "pending" + assert entries[0].synced is False + + def test_status_reports_pending_when_imported_target_was_deleted( + self, make_config, fake_lmstudio + ): + config = make_config() + service = build_service(config, fake_lmstudio) + write_manifest(config, digest=DIGEST_A) + write_blob(config, DIGEST_A) + missing_target = config.lmstudio_models_dir / "ollama" / "llama3" / "gone.gguf" + StateStore(config.state_file).upsert( + replace( + make_sync_record("llama3:1b", DIGEST_A), + imported_model_path=missing_target, + ) + ) + + entries = service.status() + + assert entries[0].display_status == "pending" + assert entries[0].synced is False + def test_select_models_rejects_unknown_name(self, make_config, fake_lmstudio): service = build_service(make_config(), fake_lmstudio) try: @@ -235,7 +409,9 @@ def test_select_models_rejects_unknown_name(self, make_config, fake_lmstudio): else: raise AssertionError("expected ValueError") - def test_select_models_matches_repository_all_tags(self, make_config, fake_lmstudio): + def test_select_models_matches_repository_all_tags( + self, make_config, fake_lmstudio + ): config = make_config() service = build_service(config, fake_lmstudio) write_manifest(config, repository="llama3", tag="1b", digest=DIGEST_A) diff --git a/tests/test_state.py b/tests/test_state.py index 171f1fd..414fa20 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1,7 +1,6 @@ import json import pytest - from conftest import DIGEST_A, make_sync_record from studiolink.models import SyncRecord @@ -10,9 +9,7 @@ def write_raw_state(path, payload, *, raw=None): path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - raw if raw is not None else json.dumps(payload), encoding="utf-8" - ) + path.write_text(raw if raw is not None else json.dumps(payload), encoding="utf-8") class TestRoundtrip: @@ -65,7 +62,9 @@ def test_unknown_link_mode_record_is_skipped(self, tmp_path): good = make_sync_record("good:1", DIGEST_A).to_json() bad = make_sync_record("bad:1", DIGEST_A).to_json() bad["link_mode"] = "nonsense-mode" - write_raw_state(path, {"schema_version": 1, "sync_records": {"bad:1": bad, "good:1": good}}) + write_raw_state( + path, {"schema_version": 1, "sync_records": {"bad:1": bad, "good:1": good}} + ) store = StateStore(path) records = store.get_all_records() @@ -75,7 +74,9 @@ def test_unknown_link_mode_record_is_skipped(self, tmp_path): def test_non_dict_record_is_skipped(self, tmp_path): path = tmp_path / "state.json" good = make_sync_record("good:1", DIGEST_A).to_json() - write_raw_state(path, {"schema_version": 1, "sync_records": {"bad:1": 42, "good:1": good}}) + write_raw_state( + path, {"schema_version": 1, "sync_records": {"bad:1": 42, "good:1": good}} + ) assert set(StateStore(path).get_all_records()) == {"good:1"} def test_missing_sync_records_key(self, tmp_path): diff --git a/tests/test_syncer.py b/tests/test_syncer.py index b3fac1f..7cf15f9 100644 --- a/tests/test_syncer.py +++ b/tests/test_syncer.py @@ -1,3 +1,4 @@ +import errno import os from conftest import ( @@ -11,7 +12,11 @@ from studiolink.models import ImportMode, ImportResult, LinkMode, ModelReadiness from studiolink.ollama_adapter import OllamaAdapter from studiolink.state import StateStore -from studiolink.syncer import Syncer, _ensure_import_alias, _format_import_message, same_file +from studiolink.syncer import ( + _ensure_import_alias, + _format_import_message, + same_file, +) def scan_models(config): @@ -46,6 +51,9 @@ def test_sync_creates_alias_and_record(self, make_syncer): record = StateStore(config.state_file).get_record("llama3:1b") assert record is not None and record.digest == DIGEST_A assert record.import_alias_path == alias + assert record.imported_model_path == ( + config.lmstudio_models_dir / model.user_repo / model.import_filename + ) def test_second_sync_skips(self, make_syncer): syncer, config, fake = make_syncer() @@ -72,6 +80,46 @@ def test_direct_mode_imports_blob_path_without_alias(self, make_syncer): assert not config.import_staging_dir.exists() +class TestCrossVolumeCopy: + def test_copy_stages_on_ollama_volume_not_state_volume( + self, make_syncer, monkeypatch + ): + syncer, config, _ = make_syncer() + write_manifest(config, digest=DIGEST_A) + write_blob(config, DIGEST_A) + model = scan_models(config)[0] + real_link = os.link + + def cross_volume_link(source, target, *args, **kwargs): + if config.import_staging_dir in target.parents: + raise OSError(errno.EXDEV, "Invalid cross-device link") + return real_link(source, target, *args, **kwargs) + + monkeypatch.setattr("studiolink.syncer.os.link", cross_volume_link) + + results = syncer.sync([model], **sync_kwargs(link_mode=LinkMode.COPY)) + + assert results[0].status == "synced" + assert results[0].record is not None + assert results[0].record.imported_model_path.is_file() + assert not (config.import_staging_dir / model.import_filename).exists() + + def test_copy_removes_reused_temporary_alias(self, make_syncer): + syncer, config, _ = make_syncer() + write_manifest(config, digest=DIGEST_A) + blob = write_blob(config, DIGEST_A) + model = scan_models(config)[0] + staging = config.ollama_models_dir / ".studiolink-imports" + staging.mkdir(parents=True) + alias = staging / model.import_filename + os.link(blob, alias) + + results = syncer.sync([model], **sync_kwargs(link_mode=LinkMode.COPY)) + + assert results[0].status == "synced" + assert not alias.exists() + + class TestUnreadyModels: def test_stale_blob_errors_with_pull_hint(self, make_syncer): syncer, config, fake = make_syncer() @@ -151,9 +199,7 @@ def test_timeout_is_isolated_per_model(self, make_syncer): write_manifest(config, digest=DIGEST_A) write_blob(config, DIGEST_A) model = scan_models(config)[0] - fake.timeout_for = { - str(config.import_staging_dir / model.import_filename) - } + fake.timeout_for = {str(config.import_staging_dir / model.import_filename)} results = syncer.sync([model], **sync_kwargs()) @@ -264,9 +310,7 @@ def test_record_saved_after_each_model(self, make_syncer): str(config.import_staging_dir / models["llama3:1b"].import_filename) } - syncer.sync( - [models["llama3:1b"], models["mistral:7b"]], **sync_kwargs() - ) + syncer.sync([models["llama3:1b"], models["mistral:7b"]], **sync_kwargs()) # Even though the batch "failed" on model 1, model 2's record persisted # after its own import.