diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..054e87b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +# Per-PR CI pipeline for grimoire. +# Runs on every pull request and push to main. +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +jobs: + lint: + name: Lint (ruff) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install ruff + run: pip install ruff + + - name: ruff check + run: ruff check src/ + + test: + name: Test (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: pip install -e ".[test]" + + - name: Run tests + run: pytest tests/ -x diff --git a/CHANGELOG.md b/CHANGELOG.md index 414af80..20fa225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,100 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] — Layered Control Plane + +This release makes grimoire composable enough to serve as THE single prompt/ +tool/skill control plane for llmcore and wairu: a builtin (shipped) pack, an +optional admin overlay, and a user overlay compose into one deterministic +view for **every** artifact type. + +### Added +- **`Grimoire.layered(roots)`** — build the facade over ordered + `(name, path, writable)` layers (lowest → highest precedence). All facade + APIs (conjure, tool_schemas, catalog, bind, validate, intent search) + resolve through the composed view. +- **`CompositeRepoView`** — a read-only merged `GrimoireRepo` over layers, + covering ALL artifact types (spells, runes, promptlets, rituals, bundles, + skilldocs, default vars). Duck-type compatible with every existing + consumer, including in-memory runtime rune registration (runtime runes + drop on `reload()` by documented design). +- **`LayeredGrimoire`** extended beyond spells: typed `get_*`/`list_*` for + all artifact types, per-key `default_vars` merge, and a generalized + `resolve_layer(artifact_id, kind)`. +- **Strict repo loading** — `GrimoireRepo.load(..., strict=True)` raises + `RepoError` on artifact parse failures and duplicate ids (fail-loud mode + for control-plane overlays); `LayeredGrimoire.from_roots(..., strict=True)` + names the failing layer. +- **`Grimoire.validate(layer=...)`** — validate a single layer in isolation + (startup validation of user overlays). +- **Caches** — static host/git conjure builtins are computed once per engine + (previously two `git` subprocesses on *every* conjure); `catalog()` and + unfiltered `tool_schemas()` are memoized with `invalidate_caches()` (called + automatically on `reload()` and by runtime rune registration). + +### Changed +- **Deterministic discovery** — artifact discovery is now `sorted()`; the + duplicate-id winner is stable (path-list order, then lexicographic file + order) instead of filesystem-dependent. +- **`grimoire.federation` imports llmcore lazily** (PEP 562) and the llmcore + requirement is declared as the `federation` extra — grimoire's core has no + llmcore dependency, so llmcore can hard-depend on grimoire with no import + cycle. + +## [0.3.0] — Procedural Tools, MCP Server, Federation & OWASP Auditing + +This release turns grimoire into a stronger prompt and tool control plane: +rune command schemas are exportable, spells and rune commands are discoverable +by intent, and a JSON-RPC MCP server exposes rune tools and prompt endpoints +to external runtimes. See `release_notes.md` for the full narrative. + +### Added — Rune command schema export +- `runes/schema.py`: centralized `param_to_json_schema`, + `command_parameters_schema`, and `command_to_openai_tool_schema` helpers; + the llmcore/wairu binders and skilldoc selector now consume these instead + of duplicating schema-building logic. + +### Added — Intent-based procedural discovery +- `Grimoire.find_by_intent(...)` (async) — rank spells against a natural-language + intent; returns `IntentMatch` results. +- `Grimoire.find_tools_by_intent(...)` (async) — same for rune commands; + returns `ToolIntentMatch` results. +- `procedural` package additions: `find_spells_by_intent_linear`, + `find_tools_by_intent_linear`, `build_spell_index_document`, + `build_rune_command_index_documents`, plus `ProceduralIndexer` / + `ProceduralRetriever` support for rune-command documents. +- Semantiscan-backed smoke test covering grimoire discovery over semantiscan. + +### Added — MCP server (JSON-RPC) +- New `mcp_server` package (auth, execution, handlers, models, server): + exposes the rune tool manifest, tool calls, and prompt endpoints over + JSON-RPC via FastAPI. +- `grimoire mcp serve` CLI command (`--host`, `--port`, `--endpoint`). +- `Grimoire.to_mcp_tool_manifest(...)` and `Grimoire.tool_schemas(...)` on the + public API. +- New `mcp` optional extra pulling in `fastapi` and `uvicorn`. + +### Added — Federation adapters +- `federation.py`: `federate_mcp_request/response/tool_call` and + `federate_rune_command(s)` adapters that normalize MCP and rune activity + into ecosystem federation events. + +### Added — OWASP metadata & auditing +- Rune contracts (rune-level and per-command) accept `owasp_categories` + metadata, preserved through parsing and carried into binder/manifest + exports and federation events. +- `grimoire rune audit` CLI command producing an OWASP coverage report + (`--filter-owasp` category filter, `--require-owasp` strict mode). + +### Added — Wairu runtime adapter +- `bind/wairu.py`: `wairu_tool_to_rune`, `wairu_tools_to_runes`, and + `register_wairu_plugin_tools` — convert wairu tool definitions back into + rune specs and register grimoire tools with a wairu plugin runtime. + +### Changed +- llmcore and wairu binders consume the centralized rune command schema + export (richer tool surface, less duplicated logic). + ## [0.2.0] — Attributes, Tag Search, Spell Write API, Layered Overlays This release adds the grimoire-side foundation for the Convergence **v0.8.0** diff --git a/pyproject.toml b/pyproject.toml index d7a820a..5564cce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "grimoire" -version = "0.3.0" +version = "0.4.0" description = "Prompt & tool control plane for the llmcore ecosystem — spells, rituals, runes." readme = "README.md" requires-python = ">=3.11" @@ -42,6 +42,13 @@ mcp = [ "fastapi>=0.110.0", "uvicorn>=0.29.0", ] +# grimoire.federation adapts rune/MCP surfaces to llmcore's ecosystem event +# envelope. Declared (not installed by default) because the hard dependency +# edge in the ecosystem points the other way: llmcore depends on grimoire. +# llmcore is not on PyPI — install from GitHub when enabling this extra. +federation = [ + "llmcore>=0.52.0", +] test = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", diff --git a/src/grimoire/api.py b/src/grimoire/api.py index 5885fc5..d982b37 100644 --- a/src/grimoire/api.py +++ b/src/grimoire/api.py @@ -61,6 +61,7 @@ import inspect import logging +from collections.abc import Sequence from pathlib import Path from typing import Any @@ -137,12 +138,74 @@ def __init__( procedural_indexer: Any | None = None, ) -> None: path = Path(repo_path) if repo_path is not None else Path.cwd() + self._layered: Any | None = None # LayeredGrimoire when built via .layered() self._repo = GrimoireRepo.load(path) self._strict = strict self._procedural_retriever = procedural_retriever self._procedural_indexer = procedural_indexer self._engine = self._build_engine() self._assembler = BundleAssembler(self._repo) + self._cache_generation = 0 + self._catalog_cache: tuple[int, dict[str, Any]] | None = None + self._tool_schemas_cache: tuple[int, list[dict[str, Any]]] | None = None + + @classmethod + def layered( + cls, + roots: Sequence[tuple[str, str | Path, bool]], + *, + strict: bool = True, + load_strict: bool = True, + scaffold_writable: bool = True, + procedural_retriever: Any | None = None, + procedural_indexer: Any | None = None, + ) -> "Grimoire": + """ + Build a Grimoire facade over ORDERED overlay layers (0.4.0). + + The composed view resolves every artifact type (spells, runes, + promptlets, rituals, bundles, skilldocs, default vars) with + deterministic precedence: **lowest → highest**, highest wins. This is + the control-plane composition used by llmcore/wairu:: + + g = Grimoire.layered([ + ("llmcore-builtin", llmcore_pack_path, False), + ("wairu-core", wairu_pack_path, False), + ("user", "~/.config/wairu/grimoire", True), + ]) + + Args: + roots: Ordered ``(name, path, writable)`` triples, lowest → + highest precedence. + strict: Conjure strictness (missing required variables raise) — + same meaning as the single-root constructor's ``strict``. + load_strict: Load layers in strict repo mode — parse failures and + duplicate ids inside a layer raise instead of log-and-skip + (fail-loud; the default for control-plane use). + scaffold_writable: Scaffold missing writable layer roots. + procedural_retriever / procedural_indexer: As in ``__init__``. + + Returns: + A layered ``Grimoire`` facade. Write APIs target the highest + writable layer via the underlying :class:`LayeredGrimoire`. + """ + from grimoire.layered import LayeredGrimoire + + layered = LayeredGrimoire.from_roots( + roots, scaffold_writable=scaffold_writable, strict=load_strict + ) + self = cls.__new__(cls) + self._layered = layered + self._repo = layered.composite_view() + self._strict = strict + self._procedural_retriever = procedural_retriever + self._procedural_indexer = procedural_indexer + self._engine = self._build_engine() + self._assembler = BundleAssembler(self._repo) + self._cache_generation = 0 + self._catalog_cache = None + self._tool_schemas_cache = None + return self # ── Internal wiring ───────────────────────────────────────────────────── @@ -157,16 +220,35 @@ def _build_engine(self) -> ConjureEngine: def reload(self) -> None: """ - Re-load the grimoire repo from disk. + Re-load the grimoire repo (or every layer) from disk. Call this after modifying files on disk to pick up changes. - All internal caches (engine, assembler) are rebuilt. + All internal caches (engine, assembler, catalog/tool-schema caches) + are rebuilt. NOTE (layered mode): artifacts registered at runtime + directly on the composed view (e.g. runtime runes) are dropped — + callers own replaying them. """ - self._repo = GrimoireRepo.load(self._repo.root) + if self._layered is not None: + self._layered.reload() + self._repo = self._layered.composite_view() + else: + self._repo = GrimoireRepo.load( + self._repo.root, writable=self._repo.writable, strict=self._repo.strict + ) self._engine = self._build_engine() self._assembler = BundleAssembler(self._repo) + self.invalidate_caches() logger.info("Grimoire reloaded from %s", self._repo.root) + def invalidate_caches(self) -> None: + """Invalidate memoized catalog/tool-schema results (0.4.0). + + Called automatically by :meth:`reload`; call manually after mutating + the repo in-memory (e.g. registering runtime runes) so cached + catalog/tool listings never serve stale entries. + """ + self._cache_generation += 1 + # ── Properties ────────────────────────────────────────────────────────── @property @@ -184,6 +266,30 @@ def version(self) -> str: """Grimoire manifest version.""" return self._repo.manifest.version + @property + def layers(self) -> list[str] | None: + """Layer names in ascending precedence, or ``None`` for single-root.""" + if self._layered is None: + return None + return [layer.name for layer in self._layered.layers] + + def resolve_layer(self, artifact_id: str, kind: str = "spell") -> str: + """ + Name of the layer an artifact resolves from (layered mode). + + Args: + artifact_id: The artifact id to locate. + kind: ``spell`` (default) | ``rune`` | ``promptlet`` | ``ritual`` + | ``bundle`` | ``skilldoc``. + + Raises: + RuntimeError: If this facade is not layered. + ArtifactNotFoundError: If no layer defines the id. + """ + if self._layered is None: + raise RuntimeError("resolve_layer() requires a layered Grimoire (.layered(...))") + return self._layered.resolve_layer(artifact_id, kind) + # ── Conjure (polymorphic) ─────────────────────────────────────────────── def conjure( @@ -463,6 +569,15 @@ def tool_schemas( if schema_format != "openai": raise ValueError(f"Unsupported tool schema format: {schema_format!r} (use 'openai')") + # Fast path (0.4.0): memoize the unfiltered listing — the hot call for + # tool inventories — keyed by the cache generation (reload/invalidate + # bumps it). Filtered calls stay uncached (cheap + rarely hot). + unfiltered = rune_ids is None and not tags + if unfiltered and self._tool_schemas_cache is not None: + gen, cached = self._tool_schemas_cache + if gen == self._cache_generation: + return cached + # Select runes if rune_ids is not None: runes = [] @@ -474,7 +589,10 @@ def tool_schemas( else: runes = self._repo.list_runes(tags=tags) - return _runes_to_openai_tools(runes) + result = _runes_to_openai_tools(runes) + if unfiltered: + self._tool_schemas_cache = (self._cache_generation, result) + return result def to_mcp_tool_manifest( self, @@ -573,15 +691,29 @@ def bind( # ── Validation & lint ─────────────────────────────────────────────────── - def validate(self) -> ValidationResult: + def validate(self, *, layer: str | None = None) -> ValidationResult: """ Run full structural validation on the grimoire repository. Checks all spells, runes, bundles, skilldocs, and cross-references. + Args: + layer: (Layered mode only) validate a SINGLE layer's repo in + isolation — required for fail-loud startup checks, where a + user overlay that merely shadows artifacts must still be + validated on its own. + Returns: ValidationResult with diagnostics. + + Raises: + RuntimeError: If ``layer`` is given on a non-layered facade. + ValueError: If ``layer`` names an unknown layer. """ + if layer is not None: + if self._layered is None: + raise RuntimeError("validate(layer=...) requires a layered Grimoire") + return validate_repo(self._layered._layer(layer).repo) return validate_repo(self._repo) def lint( @@ -619,13 +751,20 @@ def catalog(self) -> dict[str, Any]: Return a JSON-serializable catalog of all artifacts. Suitable for agent introspection and LLM tool discovery. - Delegates to ``GrimoireRepo.catalog()``. + Delegates to ``GrimoireRepo.catalog()``; memoized per cache + generation (see :meth:`invalidate_caches`). Returns: Dict with grimoire metadata, spells, runes, rituals, bundles, skilldocs, and promptlet listings. """ - return self._repo.catalog() + if self._catalog_cache is not None: + gen, cached = self._catalog_cache + if gen == self._cache_generation: + return cached + result = self._repo.catalog() + self._catalog_cache = (self._cache_generation, result) + return result # ── Convenience accessors ─────────────────────────────────────────────── diff --git a/src/grimoire/bind/wairu.py b/src/grimoire/bind/wairu.py index b013bac..f646d38 100644 --- a/src/grimoire/bind/wairu.py +++ b/src/grimoire/bind/wairu.py @@ -276,6 +276,13 @@ def register_wairu_plugin_tools( for rune in runes: engine_runes[rune.id] = rune + # 0.4.0: a Grimoire facade memoizes catalog/tool_schemas — invalidate so + # freshly registered runtime runes are immediately visible (guarded: + # bare GrimoireRepo targets have no caches). + invalidate = getattr(target, "invalidate_caches", None) + if callable(invalidate): + invalidate() + return runes diff --git a/src/grimoire/conjure/engine.py b/src/grimoire/conjure/engine.py index 021135a..4312e1b 100644 --- a/src/grimoire/conjure/engine.py +++ b/src/grimoire/conjure/engine.py @@ -92,6 +92,74 @@ def __init__( ) -> None: self._promptlets: dict[str, Promptlet] = promptlets or {} self._runes: dict[str, RuneSpec] = runes or {} + # 0.4.0: static host/git builtins are computed ONCE per engine (they + # previously ran two `git` subprocesses + os/socket lookups on EVERY + # conjure — a hot-path cost for prompt-serving control planes). Call + # refresh_builtins() to recompute (Grimoire.reload() does). + self._static_builtins: dict[str, Any] = self._compute_static_builtins() + + def refresh_builtins(self) -> None: + """Recompute the cached host/git builtin variables.""" + self._static_builtins = self._compute_static_builtins() + + @staticmethod + def _compute_static_builtins() -> dict[str, Any]: + """Compute the per-process-stable builtin variables (host + git).""" + static: dict[str, Any] = {} + + # W2: host built-ins — graceful fallback on any error + import os + import socket + + try: + static["grimoire.host.username"] = os.getlogin() + except Exception: + try: + static["grimoire.host.username"] = os.environ.get( + "USER", os.environ.get("USERNAME", "unknown") + ) + except Exception: + static["grimoire.host.username"] = "unknown" + try: + static["grimoire.host.hostname"] = socket.gethostname() + except Exception: + static["grimoire.host.hostname"] = "unknown" + try: + static["grimoire.host.cwd"] = os.getcwd() + except Exception: + static["grimoire.host.cwd"] = "unknown" + + # W2: git built-ins — requires git in PATH; graceful fallback + import subprocess + + try: + repo_name = ( + subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], + stderr=subprocess.DEVNULL, + timeout=2, + ) + .decode() + .strip() + ) + static["grimoire.git.repo_name"] = os.path.basename(repo_name) + except Exception: + static["grimoire.git.repo_name"] = "unknown" + try: + ref = ( + subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + stderr=subprocess.DEVNULL, + timeout=2, + ) + .decode() + .strip() + ) + static["grimoire.git.ref"] = ref + except Exception: + static["grimoire.git.ref"] = "unknown" + + return static # ── Public API ────────────────────────────────────────────────────────── @@ -173,7 +241,12 @@ def _build_var_map( """ effective: dict[str, Any] = {} - # Layer 1: Built-ins + # Layer 1: Built-ins. + # Static host/git builtins come from the per-engine cache (0.4.0 — + # see _compute_static_builtins); time/spell/run builtins stay + # per-call by design. + effective.update(self._static_builtins) + now = datetime.now(timezone.utc) effective["grimoire.now.iso"] = now.isoformat() effective["grimoire.now.date"] = now.strftime("%Y-%m-%d") @@ -182,58 +255,6 @@ def _build_var_map( effective["grimoire.spell.name"] = spell.name effective["grimoire.spell.version"] = spell.version - # W2: host built-ins — graceful fallback on any error - import os - import socket - - try: - effective["grimoire.host.username"] = os.getlogin() - except Exception: - try: - effective["grimoire.host.username"] = os.environ.get( - "USER", os.environ.get("USERNAME", "unknown") - ) - except Exception: - effective["grimoire.host.username"] = "unknown" - try: - effective["grimoire.host.hostname"] = socket.gethostname() - except Exception: - effective["grimoire.host.hostname"] = "unknown" - try: - effective["grimoire.host.cwd"] = os.getcwd() - except Exception: - effective["grimoire.host.cwd"] = "unknown" - - # W2: git built-ins — requires git in PATH; graceful fallback - import subprocess - - try: - repo_name = ( - subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], - stderr=subprocess.DEVNULL, - timeout=2, - ) - .decode() - .strip() - ) - effective["grimoire.git.repo_name"] = os.path.basename(repo_name) - except Exception: - effective["grimoire.git.repo_name"] = "unknown" - try: - ref = ( - subprocess.check_output( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], - stderr=subprocess.DEVNULL, - timeout=2, - ) - .decode() - .strip() - ) - effective["grimoire.git.ref"] = ref - except Exception: - effective["grimoire.git.ref"] = "unknown" - # W2: run built-ins — unique identifiers for this invocation import uuid diff --git a/src/grimoire/federation.py b/src/grimoire/federation.py index 8af3a14..f8358d9 100644 --- a/src/grimoire/federation.py +++ b/src/grimoire/federation.py @@ -10,9 +10,47 @@ from collections.abc import Iterable from dataclasses import asdict, is_dataclass from datetime import UTC, datetime -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: # pragma: no cover - annotations only + from llmcore.observability.federation import EcosystemEvent, SourceSystem + +# The llmcore federation types (EcosystemEvent, SourceSystem) are imported +# LAZILY at first use (0.4.0). Grimoire's core has NO llmcore dependency — +# llmcore depends on grimoire, and the old eager import here was the one edge +# of a potential import cycle. The envelope is an opt-in extra: +# `pip install grimoire[federation]`. +_TYPES_CACHE: tuple[Any, Any] | None = None + + +def _federation_types() -> tuple[Any, Any]: + """Import and cache (EcosystemEvent, SourceSystem), or raise clearly.""" + global _TYPES_CACHE + if _TYPES_CACHE is not None: + return _TYPES_CACHE + try: + from llmcore.observability.federation import ( + EcosystemEvent as _EcosystemEvent, + ) + from llmcore.observability.federation import ( + SourceSystem as _SourceSystem, + ) + except ImportError as e: # pragma: no cover - environment-dependent + raise ImportError( + "grimoire.federation requires llmcore — install the " + "'grimoire[federation]' extra (grimoire's core has no llmcore " + "dependency; llmcore depends on grimoire)" + ) from e + _TYPES_CACHE = (_EcosystemEvent, _SourceSystem) + return _TYPES_CACHE -from llmcore.observability.federation import EcosystemEvent, SourceSystem + +def __getattr__(name: str) -> Any: + """Serve the re-exported llmcore types lazily (PEP 562).""" + if name in ("EcosystemEvent", "SourceSystem"): + event_cls, source_cls = _federation_types() + return event_cls if name == "EcosystemEvent" else source_cls + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") def federate_mcp_request( @@ -23,6 +61,8 @@ def federate_mcp_request( ) -> EcosystemEvent: """Normalize a Grimoire MCP JSON-RPC request.""" + EcosystemEvent, SourceSystem = _federation_types() + data = _mapping(request) method = _string_value(data.get("method")) or "unknown" request_id = _string_value(data.get("id")) @@ -54,6 +94,8 @@ def federate_mcp_response( ) -> EcosystemEvent: """Normalize a Grimoire MCP JSON-RPC response.""" + EcosystemEvent, SourceSystem = _federation_types() + data = _mapping(response) error = data.get("error") has_error = error is not None @@ -89,6 +131,8 @@ def federate_mcp_tool_call( ) -> EcosystemEvent: """Normalize a Grimoire MCP tool-call attempt or result.""" + EcosystemEvent, SourceSystem = _federation_types() + meta = tool.get("_meta") if isinstance(tool.get("_meta"), dict) else {} result_meta = result.get("_meta") if isinstance(result, dict) and isinstance(result.get("_meta"), dict) else {} call_status = _string_value(result_meta.get("grimoire.call_status")) or "requested" @@ -130,6 +174,8 @@ def federate_rune_command( ) -> EcosystemEvent: """Normalize one Grimoire rune command contract.""" + EcosystemEvent, SourceSystem = _federation_types() + rune_data = _mapping(rune) command_data = _mapping(command) rune_id = _string_value(rune_data.get("id")) or "unknown" diff --git a/src/grimoire/layered.py b/src/grimoire/layered.py index 2d74990..eb3f3ef 100644 --- a/src/grimoire/layered.py +++ b/src/grimoire/layered.py @@ -39,11 +39,31 @@ import yaml from grimoire.exceptions import ArtifactNotFoundError, RepoError -from grimoire.models import Spell +from grimoire.models import ( + Bundle, + GrimoireManifest, + Promptlet, + Ritual, + RuneSpec, + SkillDoc, + Spell, +) from grimoire.store.repo import GrimoireRepo, _tag_match, _validate_match logger = logging.getLogger(__name__) +#: Artifact kinds resolvable across layers, mapped to the GrimoireRepo private +#: index attribute that stores them. This is the single source of truth for +#: which artifact types participate in layering (0.4.0: ALL of them). +_ARTIFACT_ATTRS: dict[str, str] = { + "spell": "_spells", + "rune": "_runes", + "promptlet": "_promptlets", + "ritual": "_rituals", + "bundle": "_bundles", + "skilldoc": "_skilldocs", +} + @dataclass class GrimoireLayer: @@ -63,6 +83,90 @@ class GrimoireLayer: writable: bool = False +class CompositeRepoView(GrimoireRepo): + """ + Read-only merged view over ordered grimoire layers (0.4.0, WS-G2b). + + Presents the layered composition as a single object satisfying the full + :class:`GrimoireRepo` duck type — so every existing consumer + (``ConjureEngine`` wiring, ``BundleAssembler``, ``RitualEvaluator``, + ``validate_repo``, binders, and wairu's ``register_wairu_plugin_tools``, + which mutates ``_runes`` in-memory) works unchanged against a layered + facade. + + Semantics: + - Merged indexes are materialized at construction: layers are folded + lowest → highest precedence, higher layers overwriting by id. + ``default_vars`` merge per-key (higher layer key wins). + - The merged indexes are **real mutable dicts**: in-memory runtime + registration (e.g. runtime runes) lands in this view only and is + lost on :meth:`refresh` / facade ``reload()`` — identical semantics + to today's single-repo facade. + - Writes are rejected (``writable=False``); use + :class:`LayeredGrimoire` write APIs, which target a writable layer, + then :meth:`refresh`. + - ``root`` is the highest-precedence layer's root (informational); + ``manifest`` is synthesized from the layer stack. + """ + + def __init__(self, layers: Sequence[GrimoireLayer]) -> None: + if not layers: + raise ValueError("CompositeRepoView requires at least one layer") + top = layers[-1].repo + manifest = GrimoireManifest( + name="+".join(layer.name for layer in layers), + version=top.manifest.version, + ) + super().__init__(top.root, manifest, writable=False) + self._layers: list[GrimoireLayer] = list(layers) + self.refresh() + + def refresh(self) -> None: + """Rebuild the merged indexes from the current layer repos. + + Discards any in-memory runtime registrations made directly on this + view (documented semantics — callers that register runtime artifacts + must replay them after a refresh/reload). + """ + for attr in _ARTIFACT_ATTRS.values(): + merged: dict[str, object] = {} + for layer in self._layers: # low → high; later overrides + merged.update(getattr(layer.repo, attr)) + setattr(self, attr, merged) + merged_vars: dict[str, object] = {} + for layer in self._layers: + merged_vars.update(layer.repo._default_vars) + self._default_vars = merged_vars + + def resolve_layer(self, artifact_id: str, kind: str = "spell") -> str: + """Name of the layer an artifact resolves from (highest wins). + + Note: artifacts registered at runtime directly on this view (present + in the merged index but in no layer) report ``"(runtime)"``. + + Raises: + ValueError: On an unknown ``kind``. + ArtifactNotFoundError: If no layer (nor the view) defines the id. + """ + attr = _artifact_attr(kind) + for layer in reversed(self._layers): + if artifact_id in getattr(layer.repo, attr): + return layer.name + if artifact_id in getattr(self, attr): + return "(runtime)" + raise ArtifactNotFoundError(f"{kind.capitalize()} not found in any layer: {artifact_id}") + + +def _artifact_attr(kind: str) -> str: + """Map an artifact kind to its GrimoireRepo index attribute (or raise).""" + try: + return _ARTIFACT_ATTRS[kind] + except KeyError: + raise ValueError( + f"Unknown artifact kind {kind!r} (expected one of {sorted(_ARTIFACT_ATTRS)})" + ) from None + + class LayeredGrimoire: """ Compose ordered grimoire layers into a single precedence-aware resolver. @@ -124,6 +228,7 @@ def from_roots( roots: Sequence[tuple[str, str | Path, bool]], *, scaffold_writable: bool = True, + strict: bool = False, ) -> "LayeredGrimoire": """ Build a :class:`LayeredGrimoire` from ``(name, path, writable)`` triples. @@ -134,12 +239,16 @@ def from_roots( scaffold_writable: If ``True`` (default), writable roots that do not yet exist are scaffolded via :meth:`ensure_layer_root`. Read-only roots must already exist. + strict: Load every layer repo in strict mode — any artifact parse + failure or duplicate id raises :class:`RepoError` (the + fail-loud mode control-plane consumers use for overlays). Returns: A composed :class:`LayeredGrimoire`. Raises: - RepoError: If a non-writable root does not exist. + RepoError: If a non-writable root does not exist, or (with + ``strict=True``) a layer fails strict loading. """ layers: list[GrimoireLayer] = [] for name, path, writable in roots: @@ -148,7 +257,10 @@ def from_roots( cls.ensure_layer_root(p, name=name) if not p.is_dir(): raise RepoError(f"Grimoire layer '{name}' root does not exist: {p}") - repo = GrimoireRepo.load(p, writable=writable) + try: + repo = GrimoireRepo.load(p, writable=writable, strict=strict) + except RepoError as e: + raise RepoError(f"Grimoire layer '{name}' failed to load: {e}") from e layers.append(GrimoireLayer(name=name, repo=repo, writable=writable)) return cls(layers) @@ -178,19 +290,37 @@ def _layer(self, name: str) -> GrimoireLayer: # ── Resolution / reads ───────────────────────────────────────────────────── - def _merged_spells(self) -> dict[str, Spell]: + def _merged(self, kind: str) -> dict[str, object]: """ - Merge spells across layers by id, with higher precedence overriding. + Merge one artifact kind across layers by id (higher precedence wins). + + Args: + kind: One of ``spell|rune|promptlet|ritual|bundle|skilldoc``. Returns: - Mapping ``{id: Spell}`` reflecting the resolved view. + Mapping ``{id: artifact}`` reflecting the resolved view. """ - merged: dict[str, Spell] = {} + attr = _artifact_attr(kind) + merged: dict[str, object] = {} for layer in self._layers: # low → high; later overrides - for spell in layer.repo._spells.values(): - merged[spell.id] = spell + merged.update(getattr(layer.repo, attr)) return merged + def _merged_spells(self) -> dict[str, Spell]: + """Merge spells across layers by id (kept for back-compat).""" + return self._merged("spell") # type: ignore[return-value] + + def _get(self, kind: str, artifact_id: str) -> object: + """Resolve one artifact by kind+id (highest-precedence layer wins).""" + attr = _artifact_attr(kind) + for layer in reversed(self._layers): # high → low + artifact = getattr(layer.repo, attr).get(artifact_id) + if artifact is not None: + return artifact + raise ArtifactNotFoundError( + f"{kind.capitalize()} not found in any layer: {artifact_id}" + ) + def get_spell(self, spell_id: str) -> Spell: """ Resolve a spell by id (highest-precedence layer wins). @@ -198,23 +328,111 @@ def get_spell(self, spell_id: str) -> Spell: Raises: ArtifactNotFoundError: If no layer defines the id. """ - for layer in reversed(self._layers): # high → low - spell = layer.repo._spells.get(spell_id) - if spell is not None: - return spell - raise ArtifactNotFoundError(f"Spell not found in any layer: {spell_id}") + return self._get("spell", spell_id) # type: ignore[return-value] + + def get_rune(self, rune_id: str) -> RuneSpec: + """Resolve a rune by id (highest-precedence layer wins).""" + return self._get("rune", rune_id) # type: ignore[return-value] + + def get_promptlet(self, promptlet_id: str) -> Promptlet: + """Resolve a promptlet by id (highest-precedence layer wins).""" + return self._get("promptlet", promptlet_id) # type: ignore[return-value] + + def get_ritual(self, ritual_id: str) -> Ritual: + """Resolve a ritual by id (highest-precedence layer wins).""" + return self._get("ritual", ritual_id) # type: ignore[return-value] - def resolve_layer(self, spell_id: str) -> str: + def get_bundle(self, bundle_id: str) -> Bundle: + """Resolve a bundle by id (highest-precedence layer wins).""" + return self._get("bundle", bundle_id) # type: ignore[return-value] + + def get_skilldoc(self, skilldoc_id: str) -> SkillDoc: + """Resolve a skilldoc by id (highest-precedence layer wins).""" + return self._get("skilldoc", skilldoc_id) # type: ignore[return-value] + + def list_runes( + self, tags: list[str] | None = None, *, match: str = "all" + ) -> list[RuneSpec]: + """List resolved runes, optionally filtered by tags (``match`` = all|any).""" + _validate_match(match) + runes = list(self._merged("rune").values()) + if tags: + wanted = set(tags) + runes = [r for r in runes if _tag_match(set(r.tags), wanted, match)] # type: ignore[attr-defined] + return sorted(runes, key=lambda r: r.id) # type: ignore[attr-defined,return-value] + + def list_promptlets(self) -> list[Promptlet]: + """List resolved promptlets.""" + return sorted(self._merged("promptlet").values(), key=lambda p: p.id) # type: ignore[attr-defined,return-value] + + def list_rituals( + self, tags: list[str] | None = None, *, match: str = "all" + ) -> list[Ritual]: + """List resolved rituals, optionally filtered by tags (``match`` = all|any).""" + _validate_match(match) + rituals = list(self._merged("ritual").values()) + if tags: + wanted = set(tags) + rituals = [r for r in rituals if _tag_match(set(r.tags), wanted, match)] # type: ignore[attr-defined] + return sorted(rituals, key=lambda r: r.id) # type: ignore[attr-defined,return-value] + + def list_bundles( + self, tags: list[str] | None = None, *, match: str = "all" + ) -> list[Bundle]: + """List resolved bundles, optionally filtered by tags (``match`` = all|any).""" + _validate_match(match) + bundles = list(self._merged("bundle").values()) + if tags: + wanted = set(tags) + bundles = [b for b in bundles if _tag_match(set(b.tags), wanted, match)] # type: ignore[attr-defined] + return sorted(bundles, key=lambda b: b.id) # type: ignore[attr-defined,return-value] + + def list_skilldocs( + self, tags: list[str] | None = None, *, match: str = "all" + ) -> list[SkillDoc]: + """List resolved skilldocs, optionally filtered by tags (``match`` = all|any).""" + _validate_match(match) + docs = list(self._merged("skilldoc").values()) + if tags: + wanted = set(tags) + docs = [d for d in docs if _tag_match(set(d.tags), wanted, match)] # type: ignore[attr-defined] + return sorted(docs, key=lambda d: d.id) # type: ignore[attr-defined,return-value] + + @property + def default_vars(self) -> dict[str, object]: + """Per-key merged default variables (higher layer key wins).""" + merged: dict[str, object] = {} + for layer in self._layers: + merged.update(layer.repo._default_vars) + return merged + + def resolve_layer(self, artifact_id: str, kind: str = "spell") -> str: """ - Return the name of the layer a spell resolves from. + Return the name of the layer an artifact resolves from. + + Args: + artifact_id: The artifact id to locate. + kind: Artifact kind (``spell`` default; + ``rune|promptlet|ritual|bundle|skilldoc``). Raises: + ValueError: On an unknown ``kind``. ArtifactNotFoundError: If no layer defines the id. """ + attr = _artifact_attr(kind) for layer in reversed(self._layers): - if spell_id in layer.repo._spells: + if artifact_id in getattr(layer.repo, attr): return layer.name - raise ArtifactNotFoundError(f"Spell not found in any layer: {spell_id}") + raise ArtifactNotFoundError( + f"{kind.capitalize()} not found in any layer: {artifact_id}" + ) + + def composite_view(self) -> CompositeRepoView: + """ + Build a fresh read-only :class:`CompositeRepoView` over the current + layers (the object the ``Grimoire`` facade uses as its ``_repo``). + """ + return CompositeRepoView(self._layers) def list_spells( self, tags: list[str] | None = None, *, match: str = "all" @@ -334,10 +552,12 @@ def delete_spell( # ── Maintenance ────────────────────────────────────────────────────────────── def reload(self) -> None: - """Reload every layer's repo from disk, preserving order/writability.""" + """Reload every layer's repo from disk, preserving order/writability/strictness.""" rebuilt: list[GrimoireLayer] = [] for layer in self._layers: - repo = GrimoireRepo.load(layer.repo.root, writable=layer.writable) + repo = GrimoireRepo.load( + layer.repo.root, writable=layer.writable, strict=layer.repo.strict + ) rebuilt.append(GrimoireLayer(name=layer.name, repo=repo, writable=layer.writable)) self._layers = rebuilt self._by_name = {layer.name: layer for layer in rebuilt} diff --git a/src/grimoire/store/repo.py b/src/grimoire/store/repo.py index 1915045..a84fb13 100644 --- a/src/grimoire/store/repo.py +++ b/src/grimoire/store/repo.py @@ -83,13 +83,25 @@ class GrimoireRepo: runes = repo.list_runes(tags=["devtools"]) """ - def __init__(self, root: Path, manifest: GrimoireManifest, *, writable: bool = True) -> None: + def __init__( + self, + root: Path, + manifest: GrimoireManifest, + *, + writable: bool = True, + strict: bool = False, + ) -> None: self.root = root self.manifest = manifest # Whether write operations (write/update/delete spell) are permitted on # this repo. Standalone repos default to writable; the layered overlay # system (LayeredGrimoire) marks shipped layers read-only. self.writable = writable + # Strict discovery: parse failures and duplicate ids RAISE instead of + # being logged-and-skipped. Used for fail-loud control-plane layers + # (llmcore/wairu load user overlays strict so a broken override aborts + # startup instead of silently shadowing nothing). + self.strict = strict self._spells: dict[str, Spell] = {} self._runes: dict[str, RuneSpec] = {} @@ -102,7 +114,9 @@ def __init__(self, root: Path, manifest: GrimoireManifest, *, writable: bool = T # ── Factory ───────────────────────────────────────────────────────────── @classmethod - def load(cls, path: str | Path, *, writable: bool = True) -> "GrimoireRepo": + def load( + cls, path: str | Path, *, writable: bool = True, strict: bool = False + ) -> "GrimoireRepo": """ Load a grimoire repo from disk. @@ -110,19 +124,23 @@ def load(cls, path: str | Path, *, writable: bool = True) -> "GrimoireRepo": path: Path to the grimoire root directory. writable: If ``False``, the repo rejects write/update/delete operations (used for shipped/read-only overlay layers). + strict: If ``True``, any artifact parse failure or duplicate id + raises :class:`RepoError` (fail-loud) instead of the default + log-and-skip / log-and-overwrite behavior. Returns: Populated GrimoireRepo instance. Raises: - RepoError: If the path is invalid or loading fails. + RepoError: If the path is invalid, loading fails, or (with + ``strict=True``) any artifact fails to parse / collides. """ root = Path(path).resolve() if not root.is_dir(): raise RepoError(f"Not a directory: {root}") manifest = cls._load_manifest(root) - repo = cls(root, manifest, writable=writable) + repo = cls(root, manifest, writable=writable, strict=strict) repo._discover_promptlets() repo._discover_spells() @@ -176,37 +194,61 @@ def _resolve_paths(self, path_list: list[str]) -> list[Path]: logger.debug(f"Path not found, skipping: {full}") return resolved + def _on_parse_error(self, kind: str, file: Path, exc: Exception) -> None: + """Handle an artifact parse failure per the strictness policy. + + Default: log and skip (historical behavior). Strict: raise + :class:`RepoError` naming the file and cause — the fail-loud mode + control-plane layers rely on. + """ + if self.strict: + raise RepoError(f"Failed to parse {kind} {file}: {exc}") from exc + logger.error(f"Failed to parse {kind} {file}: {exc}") + + def _on_duplicate(self, kind: str, artifact_id: str, file: Path) -> None: + """Handle a duplicate artifact id per the strictness policy. + + Default: warn and let the later file win. With deterministic (sorted) + discovery the winner is stable: last in (path-list order, then + lexicographic file order). Strict: raise :class:`RepoError`. + """ + if self.strict: + raise RepoError(f"Duplicate {kind} id '{artifact_id}' (second file: {file})") + logger.warning(f"Duplicate {kind} id '{artifact_id}', overwriting") + def _discover_spells(self) -> None: - """Find and parse all *.spell.md files.""" + """Find and parse all *.spell.md files (deterministic sorted order).""" dirs = self._resolve_paths(self.manifest.spell_paths) for d in dirs: - for spell_file in d.rglob("*.spell.md"): + for spell_file in sorted(d.rglob("*.spell.md")): try: spell = parse_spell_file(spell_file) - if spell.id in self._spells: - logger.warning(f"Duplicate spell id '{spell.id}', overwriting") - self._spells[spell.id] = spell except Exception as e: - logger.error(f"Failed to parse spell {spell_file}: {e}") + self._on_parse_error("spell", spell_file, e) + continue + if spell.id in self._spells: + self._on_duplicate("spell", spell.id, spell_file) + self._spells[spell.id] = spell def _discover_runes(self) -> None: - """Find and parse all *.rune.yaml files.""" + """Find and parse all *.rune.yaml files (deterministic sorted order).""" dirs = self._resolve_paths(self.manifest.rune_paths) for d in dirs: - for rune_file in d.rglob("*.rune.yaml"): + for rune_file in sorted(d.rglob("*.rune.yaml")): try: rune = parse_rune_file(rune_file) - if rune.id in self._runes: - logger.warning(f"Duplicate rune id '{rune.id}', overwriting") - self._runes[rune.id] = rune except Exception as e: - logger.error(f"Failed to parse rune {rune_file}: {e}") + self._on_parse_error("rune", rune_file, e) + continue + if rune.id in self._runes: + self._on_duplicate("rune", rune.id, rune_file) + self._runes[rune.id] = rune def _discover_promptlets(self) -> None: """Find and load promptlet .md files (plain markdown, no frontmatter).""" dirs = self._resolve_paths(self.manifest.promptlet_paths) for d in dirs: - for md_file in d.rglob("*.md"): + for md_file in sorted(d.rglob("*.md")): # Skip spell files if md_file.name.endswith(".spell.md"): continue @@ -215,54 +257,60 @@ def _discover_promptlets(self) -> None: promptlet_id = str(rel.with_suffix("")).replace("\\", "/") try: content = md_file.read_text(encoding="utf-8").strip() - self._promptlets[promptlet_id] = Promptlet( - id=promptlet_id, - content=content, - source_path=str(md_file), - ) except Exception as e: - logger.error(f"Failed to load promptlet {md_file}: {e}") + self._on_parse_error("promptlet", md_file, e) + continue + if promptlet_id in self._promptlets: + self._on_duplicate("promptlet", promptlet_id, md_file) + self._promptlets[promptlet_id] = Promptlet( + id=promptlet_id, + content=content, + source_path=str(md_file), + ) def _discover_rituals(self) -> None: - """Find and parse all *.ritual.yaml files.""" + """Find and parse all *.ritual.yaml files (deterministic sorted order).""" dirs = self._resolve_paths(self.manifest.ritual_paths) for d in dirs: - for ritual_file in d.rglob("*.ritual.yaml"): + for ritual_file in sorted(d.rglob("*.ritual.yaml")): try: ritual = parse_ritual_file(ritual_file) - if ritual.id in self._rituals: - logger.warning(f"Duplicate ritual id '{ritual.id}', overwriting") - self._rituals[ritual.id] = ritual except Exception as e: - logger.error(f"Failed to parse ritual {ritual_file}: {e}") + self._on_parse_error("ritual", ritual_file, e) + continue + if ritual.id in self._rituals: + self._on_duplicate("ritual", ritual.id, ritual_file) + self._rituals[ritual.id] = ritual def _discover_bundles(self) -> None: - """Find and parse all *.bundle.yaml files.""" + """Find and parse all *.bundle.yaml files (deterministic sorted order).""" dirs = self._resolve_paths(self.manifest.bundle_paths) for d in dirs: - for bundle_file in d.rglob("*.bundle.yaml"): + for bundle_file in sorted(d.rglob("*.bundle.yaml")): try: bundle = parse_bundle_file(bundle_file) - if bundle.id in self._bundles: - logger.warning(f"Duplicate bundle id '{bundle.id}', overwriting") - self._bundles[bundle.id] = bundle except Exception as e: - logger.error(f"Failed to parse bundle {bundle_file}: {e}") + self._on_parse_error("bundle", bundle_file, e) + continue + if bundle.id in self._bundles: + self._on_duplicate("bundle", bundle.id, bundle_file) + self._bundles[bundle.id] = bundle def _discover_skilldocs(self) -> None: - """Find and parse all *.skilldoc.md files.""" + """Find and parse all *.skilldoc.md files (deterministic sorted order).""" from grimoire.skilldocs.parser import parse_skilldoc_file dirs = self._resolve_paths(self.manifest.skilldoc_paths) for d in dirs: - for sd_file in d.rglob("*.skilldoc.md"): + for sd_file in sorted(d.rglob("*.skilldoc.md")): try: skilldoc = parse_skilldoc_file(sd_file) - if skilldoc.id in self._skilldocs: - logger.warning(f"Duplicate skilldoc id '{skilldoc.id}', overwriting") - self._skilldocs[skilldoc.id] = skilldoc except Exception as e: - logger.error(f"Failed to parse skilldoc {sd_file}: {e}") + self._on_parse_error("skilldoc", sd_file, e) + continue + if skilldoc.id in self._skilldocs: + self._on_duplicate("skilldoc", skilldoc.id, sd_file) + self._skilldocs[skilldoc.id] = skilldoc def _load_default_vars(self) -> None: """Load default variables from vars/defaults.yaml.""" @@ -273,7 +321,7 @@ def _load_default_vars(self) -> None: if isinstance(data, dict): self._default_vars = data except Exception as e: - logger.error(f"Failed to load default vars: {e}") + self._on_parse_error("default vars", vars_path, e) # ── Accessors ─────────────────────────────────────────────────────────── diff --git a/tests/unit/test_federation.py b/tests/unit/test_federation.py index c3f84a1..a94aa0c 100644 --- a/tests/unit/test_federation.py +++ b/tests/unit/test_federation.py @@ -1,5 +1,11 @@ from __future__ import annotations +import pytest + +# grimoire.federation imports llmcore at module level; llmcore is an optional +# ecosystem dependency (not declared by grimoire), so skip when unavailable. +pytest.importorskip("llmcore.observability.federation") + from grimoire.federation import ( federate_mcp_request, federate_mcp_response, @@ -7,7 +13,11 @@ federate_rune_command, federate_rune_commands, ) -from grimoire.mcp_server.models import JSONRPCError, JSONRPCRequest, JSONRPCResponse +from grimoire.mcp_server.models import ( + JSONRPCError, + JSONRPCRequest, + JSONRPCResponse, +) from grimoire.models import CommandSpec, Permission, RiskLevel, RuneSpec diff --git a/tests/unit/test_layered_all_types.py b/tests/unit/test_layered_all_types.py new file mode 100644 index 0000000..eeee695 --- /dev/null +++ b/tests/unit/test_layered_all_types.py @@ -0,0 +1,410 @@ +# tests/unit/test_layered_all_types.py +""" +0.4.0 layered-composition tests (WS-G2b). + +Covers the control-plane enablement work: +- LayeredGrimoire resolution for ALL artifact types (not just spells) +- CompositeRepoView satisfying the GrimoireRepo duck type +- Grimoire.layered() facade parity (conjure/tool_schemas/catalog/validate/ + resolve_layer/reload) + write rejection on the composite +- wairu-compat: runtime rune registration against a layered facade, cache + invalidation, and drop-on-reload semantics +- Deterministic + strict repo loading +- Lazy federation import (no llmcore at import time) +- Facade catalog/tool_schemas caches + engine static-builtin cache +""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from grimoire import Grimoire +from grimoire.exceptions import ArtifactNotFoundError, RepoError +from grimoire.layered import CompositeRepoView, LayeredGrimoire +from grimoire.store.repo import GrimoireRepo + +FIXTURE_REPO = Path(__file__).parent.parent / "fixtures" / "grimoire_repo" + + +# --------------------------------------------------------------------------- +# Overlay fixture: a higher-precedence layer overriding base artifacts +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def overlay_root(tmp_path: Path) -> Path: + """A small overlay repo that overrides one artifact of several types.""" + root = tmp_path / "overlay" + (root / "spells").mkdir(parents=True) + (root / "runes" / "contracts").mkdir(parents=True) + (root / "rituals").mkdir(parents=True) + (root / "spells" / "promptlets").mkdir(parents=True) + (root / "vars").mkdir(parents=True) + + (root / "grimoire.yaml").write_text( + textwrap.dedent( + """\ + name: overlay + version: 9.9.9 + spell_paths: [spells/] + rune_paths: [runes/contracts/] + ritual_paths: [rituals/] + promptlet_paths: [spells/promptlets/] + vars_path: vars/defaults.yaml + """ + ) + ) + + # Overrides the base repo's agentic/iterative_debug_loop spell id. + (root / "spells" / "debug_loop_override.spell.md").write_text( + textwrap.dedent( + """\ + --- + id: agentic/iterative_debug_loop + name: Overridden Debug Loop + version: 2.0.0 + tags: [agentic, override] + variables: + problem_statement: + type: multiline + required: true + --- + + # SYSTEM + OVERLAY WINS: {{ problem_statement }} + """ + ) + ) + + # Overrides the base repo's devtools/git rune id. + (root / "runes" / "contracts" / "git_override.rune.yaml").write_text( + textwrap.dedent( + """\ + --- + id: devtools/git + name: Git (overlay) + version: 2.0.0 + description: Overridden git rune. + tags: [devtools, override] + risk_level: medium + commands: + - name: status + summary: Overridden status + params: [] + side_effects: [] + """ + ) + ) + + # Overrides the base repo's rituals/greet_loop ritual id. + (root / "rituals" / "greet_override.ritual.yaml").write_text( + textwrap.dedent( + """\ + id: rituals/greet_loop + name: Overridden Greeting Loop + version: 2.0.0 + steps: + - id: step_1 + spell: examples/greet + conjure: + vars: + inherit: true + """ + ) + ) + + # A NEW promptlet only the overlay has. + (root / "spells" / "promptlets" / "overlay_note.md").write_text("overlay promptlet") + + # Override one default var + add a new one. + (root / "vars" / "defaults.yaml").write_text("user_name: Overlay\noverlay_only: yes\n") + + return root + + +@pytest.fixture() +def layered(overlay_root: Path) -> LayeredGrimoire: + return LayeredGrimoire.from_roots( + [("base", FIXTURE_REPO, False), ("overlay", overlay_root, False)], + scaffold_writable=False, + ) + + +# --------------------------------------------------------------------------- +# LayeredGrimoire: all-artifact-type resolution +# --------------------------------------------------------------------------- + + +class TestLayeredAllTypes: + def test_spell_override_wins(self, layered: LayeredGrimoire): + spell = layered.get_spell("agentic/iterative_debug_loop") + assert spell.name == "Overridden Debug Loop" + assert layered.resolve_layer("agentic/iterative_debug_loop", "spell") == "overlay" + + def test_rune_override_wins(self, layered: LayeredGrimoire): + rune = layered.get_rune("devtools/git") + assert rune.name == "Git (overlay)" + assert str(rune.risk_level) in ("RiskLevel.MEDIUM", "medium") + assert layered.resolve_layer("devtools/git", "rune") == "overlay" + + def test_rune_from_base_still_resolves(self, layered: LayeredGrimoire): + rune = layered.get_rune("wairu/shell") + assert rune is not None + assert layered.resolve_layer("wairu/shell", "rune") == "base" + + def test_ritual_override_wins(self, layered: LayeredGrimoire): + ritual = layered.get_ritual("rituals/greet_loop") + assert ritual.name == "Overridden Greeting Loop" + assert layered.resolve_layer("rituals/greet_loop", "ritual") == "overlay" + + def test_promptlet_merge(self, layered: LayeredGrimoire): + assert layered.get_promptlet("overlay_note").content == "overlay promptlet" + # Base promptlets still visible. + assert layered.get_promptlet("safety/base_engineering") is not None + + def test_bundle_and_skilldoc_passthrough(self, layered: LayeredGrimoire): + base = GrimoireRepo.load(FIXTURE_REPO) + assert {b.id for b in layered.list_bundles()} == set(base._bundles) + assert {d.id for d in layered.list_skilldocs()} == set(base._skilldocs) + + def test_default_vars_merge_per_key(self, layered: LayeredGrimoire): + merged = layered.default_vars + assert merged["user_name"] == "Overlay" # overlay key wins + assert merged["overlay_only"] is True or merged["overlay_only"] == "yes" + + def test_unknown_kind_raises(self, layered: LayeredGrimoire): + with pytest.raises(ValueError, match="Unknown artifact kind"): + layered.resolve_layer("x", "wand") + + def test_missing_artifact_raises(self, layered: LayeredGrimoire): + with pytest.raises(ArtifactNotFoundError): + layered.get_rune("no/such_rune") + + +# --------------------------------------------------------------------------- +# CompositeRepoView + Grimoire.layered() facade parity +# --------------------------------------------------------------------------- + + +class TestLayeredFacade: + def test_composite_view_duck_type(self, layered: LayeredGrimoire): + view = layered.composite_view() + assert isinstance(view, CompositeRepoView) + assert isinstance(view, GrimoireRepo) # the load-bearing property + assert view.writable is False + assert view.get_rune("devtools/git").name == "Git (overlay)" + assert "agentic/iterative_debug_loop" in view._spells + assert view.default_vars["user_name"] == "Overlay" + assert view.resolve_layer("devtools/git", "rune") == "overlay" + + def test_composite_rejects_writes(self, layered: LayeredGrimoire): + view = layered.composite_view() + spell = view.get_spell("agentic/iterative_debug_loop") + with pytest.raises(RepoError, match="read-only"): + view.write_spell(spell, overwrite=True) + + def test_facade_layered_reads_and_conjure(self, overlay_root: Path): + g = Grimoire.layered( + [("base", FIXTURE_REPO, False), ("overlay", overlay_root, False)] + ) + assert g.layers == ["base", "overlay"] + assert g.resolve_layer("devtools/git", "rune") == "overlay" + # Conjure resolves the OVERLAY spell body. + result = g.conjure_spell( + "agentic/iterative_debug_loop", variables={"problem_statement": "X"} + ) + text = "\n".join(block.content for block in result.blocks) + assert "OVERLAY WINS: X" in text + + def test_facade_tool_schemas_reflect_overlay(self, overlay_root: Path): + g = Grimoire.layered( + [("base", FIXTURE_REPO, False), ("overlay", overlay_root, False)] + ) + schemas = g.tool_schemas() + names = [s["function"]["name"] for s in schemas] + # Overlay git rune has exactly ONE command (status): the base rune's + # other commands must NOT leak through. + git_tools = [n for n in names if n.startswith("devtools__git__")] + assert git_tools == ["devtools__git__status"] + + def test_facade_validate_layer(self, overlay_root: Path): + g = Grimoire.layered( + [("base", FIXTURE_REPO, False), ("overlay", overlay_root, False)] + ) + result = g.validate(layer="overlay") + assert result is not None + with pytest.raises(ValueError): + g.validate(layer="nope") + + def test_single_root_facade_rejects_layer_apis(self): + g = Grimoire(FIXTURE_REPO) + assert g.layers is None + with pytest.raises(RuntimeError): + g.resolve_layer("devtools/git", "rune") + with pytest.raises(RuntimeError): + g.validate(layer="base") + + def test_wairu_runtime_rune_registration_and_reload(self, overlay_root: Path): + """wairu-compat: runtime runes land in the composite view, are served + through (cache-invalidated) tool_schemas, and drop on reload().""" + from grimoire.bind.wairu import register_wairu_plugin_tools + + g = Grimoire.layered( + [("base", FIXTURE_REPO, False), ("overlay", overlay_root, False)] + ) + _ = g.tool_schemas() # prime the cache + + class FakeTool: + name = "frobnicate" + description = "Frobnicates things" + parameters: dict = {"type": "object", "properties": {}} # noqa: RUF012 + requires_approval = False + risk_level = "low" + + runes = register_wairu_plugin_tools(g, [FakeTool()], plugin_name="testplug") + assert len(runes) == 1 + names = [s["function"]["name"] for s in g.tool_schemas()] + assert any("frobnicate" in n for n in names), names # cache invalidated + + g.reload() # runtime runes are documented to drop on reload + names_after = [s["function"]["name"] for s in g.tool_schemas()] + assert not any("frobnicate" in n for n in names_after) + # Layered state survives the reload. + assert g.resolve_layer("devtools/git", "rune") == "overlay" + + +# --------------------------------------------------------------------------- +# Deterministic + strict loading +# --------------------------------------------------------------------------- + + +def _write_min_repo(root: Path, spell_bodies: dict[str, str]) -> None: + (root / "spells").mkdir(parents=True) + (root / "grimoire.yaml").write_text("name: t\nspell_paths: [spells/]\n") + for fname, body in spell_bodies.items(): + (root / "spells" / fname).write_text(body) + + +def _spell(spell_id: str, name: str) -> str: + return textwrap.dedent( + f"""\ + --- + id: {spell_id} + name: {name} + --- + + # USER + hello + """ + ) + + +class TestDeterministicStrictLoad: + def test_duplicate_last_lexicographic_wins(self, tmp_path: Path): + _write_min_repo( + tmp_path, + { + "a_first.spell.md": _spell("dup/id", "First"), + "z_last.spell.md": _spell("dup/id", "Last"), + }, + ) + repo = GrimoireRepo.load(tmp_path) # non-strict: warn + overwrite + assert repo.get_spell("dup/id").name == "Last" + + def test_strict_duplicate_raises(self, tmp_path: Path): + _write_min_repo( + tmp_path, + { + "a_first.spell.md": _spell("dup/id", "First"), + "z_last.spell.md": _spell("dup/id", "Last"), + }, + ) + with pytest.raises(RepoError, match="Duplicate spell id"): + GrimoireRepo.load(tmp_path, strict=True) + + def test_strict_parse_failure_raises(self, tmp_path: Path): + _write_min_repo(tmp_path, {"broken.spell.md": "not: [valid frontmatter"}) + with pytest.raises(RepoError, match="Failed to parse spell"): + GrimoireRepo.load(tmp_path, strict=True) + # Non-strict: skipped with a logged error. + repo = GrimoireRepo.load(tmp_path) + assert repo._spells == {} + + def test_from_roots_strict_names_layer(self, tmp_path: Path): + _write_min_repo(tmp_path / "bad", {"broken.spell.md": "not: [valid"}) + with pytest.raises(RepoError, match="layer 'user' failed to load"): + LayeredGrimoire.from_roots( + [("base", FIXTURE_REPO, False), ("user", tmp_path / "bad", False)], + scaffold_writable=False, + strict=True, + ) + + +# --------------------------------------------------------------------------- +# Lazy federation import (cycle hazard) +# --------------------------------------------------------------------------- + + +class TestLazyFederation: + def test_import_federation_does_not_import_llmcore(self): + """grimoire.federation must not pull llmcore at import time — llmcore + depends on grimoire (0.4.0+), so this edge must stay lazy.""" + code = ( + "import sys\n" + "import grimoire.federation\n" + "bad = [m for m in sys.modules if m.split('.')[0] == 'llmcore']\n" + "assert not bad, f'llmcore imported eagerly: {bad}'\n" + "print('lazy OK')\n" + ) + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, timeout=60 + ) + assert proc.returncode == 0, proc.stderr + assert "lazy OK" in proc.stdout + + def test_reexport_still_works(self): + # llmcore IS installed in this venv; the lazy re-export must serve it. + from grimoire.federation import EcosystemEvent, SourceSystem + + assert EcosystemEvent is not None + assert SourceSystem is not None + + +# --------------------------------------------------------------------------- +# Caches +# --------------------------------------------------------------------------- + + +class TestCaches: + def test_catalog_memoized_and_invalidated(self): + g = Grimoire(FIXTURE_REPO) + first = g.catalog() + assert g.catalog() is first # memoized + g.invalidate_caches() + assert g.catalog() is not first + + def test_tool_schemas_unfiltered_memoized(self): + g = Grimoire(FIXTURE_REPO) + first = g.tool_schemas() + assert g.tool_schemas() is first + # Filtered calls bypass the cache. + filtered = g.tool_schemas(tags=["devtools"]) + assert filtered is not first + g.reload() + assert g.tool_schemas() is not first + + def test_engine_static_builtins_cached(self): + g = Grimoire(FIXTURE_REPO) + engine = g._engine + snapshot = engine._static_builtins + # Conjuring must not recompute the static builtin dict. + spell = g.get_spell("agentic/iterative_debug_loop") + engine.conjure(spell, variables={"problem_statement": "x"}, strict=False) + assert engine._static_builtins is snapshot + engine.refresh_builtins() + assert engine._static_builtins is not snapshot