From 0c7e90dc63a13304f6db4ebc785265fcebe4a88d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 15:08:29 -0400 Subject: [PATCH 1/2] Harden Axiom adapter references and kernel hash coverage Reject submodules, repository-environment redirection, and changed resolved module paths. Check committed paths, unfiltered bytes and executable modes, including edits hidden from Git status. Clarify array and checkout claims. Property-test compile pins, checkout integrity and reference-name-independent kernel hashes, with same-qualname adapters from different modules and Git blob differential checks. Both reviewed hash-order mutants are killed. Validation: 183 passed, 4 real-engine skips in the two targeted files; main regressions reproduced, lint/format/test-plan checks passed. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: GPT-6.1 Sol --- changelog.d/axiom-adapter-hardening.fixed.md | 15 + changelog.d/axiom-graph-adapter.added.md | 4 +- .../src/microcosm/frame/adapters/axiom.py | 295 +++++-- .../shared/test_axiom_graph_adapter.py | 738 +++++++++++++++++- .../engine_free/shared/test_rules_kernels.py | 201 ++++- 5 files changed, 1201 insertions(+), 52 deletions(-) create mode 100644 changelog.d/axiom-adapter-hardening.fixed.md diff --git a/changelog.d/axiom-adapter-hardening.fixed.md b/changelog.d/axiom-adapter-hardening.fixed.md new file mode 100644 index 0000000000..5dd9229b54 --- /dev/null +++ b/changelog.d/axiom-adapter-hardening.fixed.md @@ -0,0 +1,15 @@ +Harden the Axiom adapter's engine references (follow-ups to microcosm#1118). +`axiom_engine_ref` now refuses a git checkout root that contains a submodule, +and it examines the root's own repository: git runs without the caller's +`GIT_DIR`, `GIT_WORK_TREE` and the other `git rev-parse --local-env-vars` +variables and without replace refs, and the root must be its repository's top +level. The files outside the root's `.git` must also be exactly the declared +commit's, each byte for byte the blob the commit records and with its +executable bit, so a change that +`git status` does not report is refused: an edit hidden from a stat cache +trusted under `core.trustctime=false`, by a clean filter or by a file system +monitor, a nested `.git` entry, or a symbolic link held as a plain file under +`core.symlinks=false`, or an executable-bit change hidden by +`core.filemode=false`. A referenced adapter also records the file its module +path resolves to, and refuses to compile or take a second reference once the +path resolves to another file. diff --git a/changelog.d/axiom-graph-adapter.added.md b/changelog.d/axiom-graph-adapter.added.md index 52a2de0504..1ca715b985 100644 --- a/changelog.d/axiom-graph-adapter.added.md +++ b/changelog.d/axiom-graph-adapter.added.md @@ -13,7 +13,9 @@ and date outputs. The default output stays as before. digest, the RuleSpec commit, the module and whole-root content digests, and the adapter configuration, so a RuleSpec edit re-keys every node that names it. It also pins the adapter, which then refuses to compile from a root whose bytes -have moved. A git-checkout root must hold exactly the declared commit. +have moved. A git checkout root must be clean, at the declared commit, +contain no submodules, and have no tracked file flagged skip-worktree or +assume-unchanged. `assert_no_relations` refuses relation-bearing modules. The new `simulate.rules_by_ref@1` kernel (`microcosm.frame.rules_kernels`) runs diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py index 1689fbc383..3b9f3b5ee6 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py @@ -68,9 +68,11 @@ Graph output types ------------------ -By default :meth:`AxiomEngine.materialize` returns the arrays the dense -surface returns: judgments as int8 codes, text and dates as Python string -lists. A graph node may own only a closed set of dtypes, so +By default :meth:`AxiomEngine.materialize` returns each dense output as a +numpy array (``np.asarray`` of what the surface returns), with no cast: +judgments as int8 codes, and text and dates (which the surface returns as +Python string lists) as numpy string arrays. A graph node may own only a +closed set of dtypes, so ``output_dtypes="graph"`` casts each output to one of them without loss — judgments to int64 codes, integers to int64, decimals to float64, booleans to bool — and refuses text and date outputs before the engine runs. @@ -86,12 +88,14 @@ Any edit to a RuleSpec byte therefore moves the reference, and with it every node key that names it. The reference also pins the adapter: from then on it compiles only from a root whose digest still matches and that holds no -symbolic link, so a node keyed by the reference is never computed from other -bytes. +symbolic link, and only when its module path still resolves to the file the +reference named, so a node keyed by the reference is never computed from +other bytes. """ import hashlib import json +import os import re import subprocess from collections.abc import Mapping, Sequence @@ -370,8 +374,10 @@ def __init__( self._programs: dict[str, Any] = {} self._metadata: dict[str, Any] | None = None # Set by axiom_engine_ref: the RuleSpec tree digest this adapter's - # reference names. Every later compile re-hashes the root against it. + # reference names, and the file its module path resolved to. Every + # later compile re-checks both. self._pinned_tree_sha256: str | None = None + self._pinned_module: Path | None = None # ------------------------------------------------------------------ # Variable metadata @@ -812,15 +818,26 @@ def _require_pinned_tree(self) -> None: """Refuse to compile when the root no longer holds the referenced bytes. Once :func:`axiom_engine_ref` has named this adapter's RuleSpec tree, - every compile refuses a symbolic link under the root (the digest does - not descend into a linked directory, so a directory or dangling link - added after the reference would not move it) and re-hashes the root, - so a node keyed by that reference is never computed from other bytes. + every compile requires the module path to resolve to the file the + reference named (a module given through a link outside the root could + be repointed at another file under it without moving the digest), + refuses a symbolic link under the root (the digest does not descend + into a linked directory, so a directory or dangling link added after + the reference would not move it) and re-hashes the root, so a node + keyed by that reference is never computed from other bytes. Raises: - ValueError: If the root holds a symbolic link, or its digest moved - since the reference. + ValueError: If the module path resolves to another file, the root + holds a symbolic link, or its digest moved since the + reference. """ + module = self._module.resolve() + if module != self._pinned_module: + raise ValueError( + f"Module {self._module} resolves to {module}, not the file " + f"{self._pinned_module} this adapter's engine_ref named; " + "construct a new adapter and reference." + ) _refuse_symlinks(self._rulespec_roots[0].resolve()) current, _ = rulespec_tree_digest(self._rulespec_roots[0].resolve()) if current != self._pinned_tree_sha256: @@ -1163,21 +1180,28 @@ def axiom_engine_ref( declaration, checked only for a git checkout root (below). The directory digest is the authority. - A git checkout root is accepted only when clean, with no untracked or - ignored files and no tracked file flagged skip-worktree or - assume-unchanged (``git status`` does not report edits to those), and at - ``rulespec_commit``. Its ``.git`` is then hashed + A git checkout root is accepted only when it is the top level of its own + repository, clean, at ``rulespec_commit``, contains no submodules, has no + tracked file flagged skip-worktree or assume-unchanged (``git status`` + does not report edits to those), and holds outside ``.git`` exactly the + commit's files, each byte for byte the blob the commit records and with + its executable bit. Git runs + without the caller's repository-selecting variables (``GIT_DIR``, + ``GIT_WORK_TREE`` and the rest of ``git rev-parse --local-env-vars``) and + without replace refs, so neither can point the check at other objects. + Its ``.git`` is then hashed like any other file, as the graph's source key hashes it. That makes the reference specific to one clone: two clones or worktrees of one commit (a worktree's ``.git`` file names an absolute path) and an export of that commit all give different references, and any later git operation in the checkout moves it. Prefer the export. - The first call pins the adapter to the tree digest it names: it refuses - an adapter that has already compiled (those programs were read from bytes - no reference names), and from then on every compile, and every later - reference to the same adapter, refuses a root whose digest has moved or - that holds a symbolic link. + The first call pins the adapter to the tree digest it names and to the + file its module path resolves to: it refuses an adapter that has already + compiled (those programs were read from bytes no reference names), and + from then on every compile, and every later reference to the same + adapter, refuses a root whose digest has moved or that holds a symbolic + link, and a module path that now resolves to another file. Args: engine: The adapter the reference names. @@ -1198,10 +1222,11 @@ def axiom_engine_ref( ValueError: If a pin is malformed; ``rulespec_root`` is not the adapter's only root; the module is not a file under it; the root contains a symbolic link (the digest would not cover what it - points at); a git checkout root is dirty, has a file flagged - skip-worktree or assume-unchanged, or is at another commit; or - the adapter compiled before its first reference, or its root moved - after it. + points at); a git checkout root is not its repository's top + level, is dirty, contains a submodule, has a file flagged + skip-worktree or assume-unchanged, holds bytes its commit does + not, or is at another commit; or the adapter compiled before its + first reference, or its root or module moved after it. """ if not isinstance(engine, AxiomEngine): raise TypeError(f"{engine!r} is not an AxiomEngine.") @@ -1247,12 +1272,19 @@ def axiom_engine_ref( "adapter." ) engine._pinned_tree_sha256 = tree_sha256 + engine._pinned_module = module elif engine._pinned_tree_sha256 != tree_sha256: raise ValueError( f"RuleSpec root {root} changed after this adapter's engine_ref named " f"it (tree digest {engine._pinned_tree_sha256} -> {tree_sha256}); " "construct a new adapter and reference." ) + elif engine._pinned_module != module: + raise ValueError( + f"Module {engine._module} resolves to {module}, not the file " + f"{engine._pinned_module} this adapter's engine_ref named; " + "construct a new adapter and reference." + ) schema = engine._schema periods = ( None @@ -1512,27 +1544,56 @@ def _refuse_symlinks(root: Path) -> None: def _require_clean_checkout(root: Path, commit: str) -> None: """Require a git-checkout RuleSpec root to hold exactly ``commit``. - Ignored files count as changes: the digest hashes them, but the commit - does not hold them. A tracked file flagged skip-worktree or - assume-unchanged is refused even when unedited, because ``git status`` - does not report edits to it. ``--no-optional-locks`` stops ``git status`` - refreshing the index, which would otherwise rewrite ``.git/index`` and - move the digest taken next. + Git examines the root's own repository: it runs without the caller's + repository-selecting variables (:func:`_git_environment`) and without + replace refs (a ``refs/replace`` entry makes one commit read as another), + and the root must be the repository's top level (``core.worktree`` can + point a repository at another directory). ``--no-optional-locks`` stops + ``git status`` refreshing the index, which would otherwise rewrite + ``.git/index`` and move the digest taken next. + + The index may hold no submodule: the commit records a submodule's + commit, not its files, and ``git status`` does not report every change + inside one. No tracked file may be flagged skip-worktree or + assume-unchanged, even unedited, because ``git status`` does not report + edits to it. Ignored files count as changes: the digest hashes them, but + the commit does not hold them. Last, the files outside the top-level + ``.git`` must be exactly the commit's, each byte for byte the regular-file + blob the commit records at its path, with its recorded executable bit. + That refuses a change ``git status`` + does not report: an edit hidden from a stat cache trusted under + ``core.trustctime=false``, by a clean filter, or by a file system monitor; + a nested ``.git`` entry, which git's untracked scan skips; or a committed + symbolic link held as a plain file under ``core.symlinks=false``; or an + executable-bit change hidden by ``core.filemode=false``. Raises: - ValueError: If git fails or times out, the checkout has modified, - untracked, or ignored files or a file flagged skip-worktree or - assume-unchanged, or ``HEAD`` is not ``commit``. + ValueError: If git fails or times out; the root is not its + repository's top level; the index holds a submodule or a file + flagged skip-worktree or assume-unchanged; the checkout has + modified, untracked, or ignored files; ``HEAD`` is not + ``commit``; or the files outside ``.git`` are not the commit's. """ + environment = _git_environment() + def git(*args: str) -> str: try: completed = subprocess.run( - ["git", "--no-optional-locks", "-C", str(root), *args], + [ + "git", + "--no-optional-locks", + "--no-replace-objects", + "-C", + str(root), + *args, + ], capture_output=True, - text=True, + encoding="utf-8", + errors="surrogateescape", check=False, timeout=120, + env=environment, ) except FileNotFoundError as exc: raise ValueError( @@ -1549,19 +1610,38 @@ def git(*args: str) -> str: ) return completed.stdout - if git("status", "--porcelain=v1", "--untracked-files=all", "--ignored").strip(): + toplevel = Path(git("rev-parse", "--show-toplevel").removesuffix("\n")) + try: + same = toplevel.samefile(root) + except OSError: + same = False + if not same: raise ValueError( - f"RuleSpec checkout {root} has modified, untracked, or ignored " - f"files that commit {commit} does not hold; reference a git " - "archive export instead." + f"RuleSpec checkout {root} is not the top level of its git " + f"repository, whose work tree is {toplevel}; reference a git archive " + "export instead." + ) + submodules: list[str] = [] + flagged: list[str] = [] + for entry in git("ls-files", "--stage", "-v", "-z").split("\0"): + if not entry: + continue + # " \t". A submodule has mode + # 160000; -v tags a skip-worktree file "S" and an assume-unchanged + # file in lower case. + fields, path = entry.split("\t", 1) + tag, mode = fields.split(" ")[:2] + if mode == "160000": + submodules.append(path) + if tag == "S" or tag.islower(): + flagged.append(path) + if submodules: + raise ValueError( + f"RuleSpec checkout {root} contains the submodule {submodules[0]} " + f"({len(submodules)} submodule(s)); commit {commit} records a " + "submodule's commit, not its files, and git status does not report " + "every change inside one. Reference a git archive export instead." ) - # ``ls-files -v`` tags a skip-worktree file ``S`` and an assume-unchanged - # file in lower case; ``git status`` reports no edit to either. - flagged = [ - entry[2:] - for entry in git("ls-files", "-v", "-z").split("\0") - if entry and (entry[0] == "S" or entry[0].islower()) - ] if flagged: raise ValueError( f"RuleSpec checkout {root}: {flagged[0]} is flagged skip-worktree " @@ -1569,9 +1649,130 @@ def git(*args: str) -> str: "status does not report edits to it; clear the flags or reference " "a git archive export instead." ) + if git("status", "--porcelain=v1", "--untracked-files=all", "--ignored").strip(): + raise ValueError( + f"RuleSpec checkout {root} has modified, untracked, or ignored " + f"files that commit {commit} does not hold; reference a git " + "archive export instead." + ) head = git("rev-parse", "--verify", "HEAD^{commit}").strip() if head != commit: raise ValueError( f"RuleSpec checkout {root} is at {head}, not the declared " f"rulespec_commit {commit}." ) + algorithm = git("rev-parse", "--show-object-format").strip() + if algorithm not in ("sha1", "sha256"): + raise ValueError( + f"RuleSpec checkout {root} uses the git object format {algorithm!r}, " + "which this check cannot hash." + ) + recorded: dict[str, tuple[str, str]] = {} + for entry in git("ls-tree", "-r", "-z", "--full-tree", commit).split("\0"): + if not entry: + continue + # " \t" + fields, path = entry.split("\t", 1) + mode, _, name = fields.split(" ") + recorded[path] = (mode, name) + present = _checkout_files(root) + unexpected = sorted(present.keys() ^ recorded.keys()) + if unexpected: + first = unexpected[0] + state = "holds" if first in present else "lacks" + raise ValueError( + f"RuleSpec checkout {root} {state} {first}, unlike commit {commit} " + f"({len(unexpected)} path(s) differ), though git status reports no " + "change; reference a git archive export instead." + ) + for path, (mode, name) in sorted(recorded.items()): + # A symbolic link checked out under core.symlinks=false is a plain + # file holding the link's blob, and git status reports no change. + if mode not in ("100644", "100755"): + raise ValueError( + f"RuleSpec checkout {root}: commit {commit} records {path} with " + f"mode {mode}, not as a regular file, though the checkout holds " + "a plain file there; reference a git archive export instead." + ) + file = present[path] + if not file.is_file() or _git_blob_id(file.read_bytes(), algorithm) != name: + raise ValueError( + f"RuleSpec checkout {root}: {path} is not the blob {name} commit " + f"{commit} records for it, though git status reports no change; " + "reference a git archive export instead." + ) + # Git records the owner's executable bit; core.filemode=false can + # hide a change to it from git status. + actual_mode = "100755" if file.stat().st_mode & 0o100 else "100644" + if actual_mode != mode: + raise ValueError( + f"RuleSpec checkout {root}: {path} has mode {actual_mode}, not " + f"the mode {mode} commit {commit} records for it, though git " + "status reports no change; reference a git archive export " + "instead." + ) + + +def _git_environment() -> dict[str, str]: + """This process's environment without the variables that choose git's repository. + + The names removed are the installed git's own list of variables local to + one repository (``git rev-parse --local-env-vars``): ``GIT_DIR``, + ``GIT_WORK_TREE``, ``GIT_INDEX_FILE``, ``GIT_OBJECT_DIRECTORY``, + ``GIT_COMMON_DIR``, the ``git -c`` carriers ``GIT_CONFIG_PARAMETERS`` and + ``GIT_CONFIG_COUNT``, and the rest. Each names the repository, index, + object store, or work tree git reads, or carries configuration into it. + + Raises: + ValueError: If git is missing, fails, or times out. + """ + + try: + completed = subprocess.run( + ["git", "rev-parse", "--local-env-vars"], + capture_output=True, + text=True, + check=False, + timeout=120, + ) + except FileNotFoundError as exc: + raise ValueError("git is required to verify a RuleSpec checkout.") from exc + except subprocess.TimeoutExpired as exc: + raise ValueError("git rev-parse --local-env-vars timed out.") from exc + if completed.returncode != 0: + raise ValueError( + f"git rev-parse --local-env-vars failed: {completed.stderr.strip()}" + ) + local = frozenset(completed.stdout.split()) + return {name: value for name, value in os.environ.items() if name not in local} + + +def _checkout_files(root: Path) -> dict[str, Path]: + """Every non-directory entry under ``root`` except the top-level ``.git``. + + Keyed by POSIX path relative to ``root``, the form ``git ls-tree`` prints. + """ + + present: dict[str, Path] = {} + for directory, directories, files in root.walk(): + if directory == root: + directories[:] = [name for name in directories if name != ".git"] + files = [name for name in files if name != ".git"] + for name in files: + path = directory / name + present[path.relative_to(root).as_posix()] = path + return present + + +def _git_blob_id(content: bytes, algorithm: str) -> str: + """The object name git gives ``content`` as a blob (no filters applied). + + Args: + content: The blob's bytes. + algorithm: The repository's object format, ``"sha1"`` or ``"sha256"``. + """ + + digest = hashlib.new(algorithm) + digest.update(b"blob %d\0" % len(content)) + digest.update(content) + return digest.hexdigest() diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py b/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py index 609e971cb1..e2477c9055 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py @@ -22,7 +22,7 @@ import shutil import subprocess import tarfile -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from pathlib import Path import numpy as np @@ -32,6 +32,7 @@ from hypothesis import strategies as st from microcosm.frame import ExportContract, Frame, WeightKind, Weights +from microcosm.frame.adapters import axiom as axiom_adapter from microcosm.frame.adapters.axiom import ( BE_SCHEMA, NZ_NESTING, @@ -317,6 +318,7 @@ def from_file(cls, path, *, rulespec_roots, entity): f"`{entity}`" ) program = _FakeProgram(entity, relations) + program.module_path = Path(path) programs[entity] = program return program @@ -832,6 +834,21 @@ def test_native_default_returns_the_engine_arrays_unchanged( assert results["family_assets_ok"].dtype == np.dtype(np.int8) assert results["family_category"].tolist() == ["small"] * 3 + def test_native_outputs_are_numpy_arrays_text_and_dates_included( + self, monkeypatch + ) -> None: + # The module docstring's claim: every output, text and dates + # included, comes back as a numpy array, uncast. + adapter = _nz_adapter() + _fake_engine(adapter, monkeypatch) + variables = ["family_assets_ok", "family_category", "family_review_date"] + results = adapter.materialize(_fake_frame(), variables, 2026) + assert all(isinstance(results[name], np.ndarray) for name in variables) + assert results["family_assets_ok"].dtype == np.dtype(np.int8) + assert results["family_category"].dtype.kind == "U" + assert results["family_review_date"].dtype.kind == "U" + assert results["family_review_date"].tolist() == ["2026-04-01"] * 3 + def test_graph_mode_returns_graph_ownable_dtypes(self, monkeypatch) -> None: adapter = _nz_adapter(output_dtypes="graph", periods={"2026-27": TAX_YEAR}) _fake_engine(adapter, monkeypatch) @@ -932,6 +949,7 @@ def test_only_axiom_engines_are_accepted(self) -> None: # ---------------------------------------------------------------------- _MODULE = "zz/policies/tests/toy.yaml" +_OTHER_MODULE = "zz/policies/tests/other.yaml" _BASE_TREE = { _MODULE: b"format: rulespec/v1\nrules: []\n", "zz/policies/shared/rates.yaml": b"format: rulespec/v1\nrules:\n - name: r\n", @@ -1308,6 +1326,155 @@ def test_a_reference_after_compiling_is_refused( with pytest.raises(ValueError, match="before the adapter compiles"): self._reference(adapter, root) + def _linked_adapter(self, tmp_path: Path) -> tuple[AxiomEngine, Path, Path]: + """An adapter whose module path is a link outside the root.""" + + root = _write_tree( + tmp_path / "rulespec", + {**_BASE_TREE, _OTHER_MODULE: b"format: rulespec/v1\n"}, + ) + link = tmp_path / "links" / "module.yaml" + link.parent.mkdir() + link.symlink_to(root / _MODULE) + return self._adapter_at(link, root), root, link + + def _adapter_at(self, module: Path, root: Path) -> AxiomEngine: + return AxiomEngine( + module, schema=NZ_SCHEMA, rulespec_roots=(root,), output_dtypes="graph" + ) + + def test_a_module_link_left_alone_compiles_the_referenced_file( + self, tmp_path, monkeypatch + ) -> None: + adapter, root, link = self._linked_adapter(tmp_path) + programs = _fake_engine(adapter, monkeypatch) + reference = self._reference(adapter, root) + assert json.loads(reference)["module"] == _MODULE + adapter.materialize(_fake_frame(), ["family_assistance"], 2026) + assert {program.module_path for program in programs.values()} == {link} + assert self._reference(adapter, root) == reference + + def test_a_module_link_repointed_after_the_reference_blocks_the_next_compile( + self, tmp_path, monkeypatch + ) -> None: + # The review's scenario: the link lies outside the root, so repointing + # it at another file under the root moves no digest the pin checks. + adapter, root, link = self._linked_adapter(tmp_path) + programs = _fake_engine(adapter, monkeypatch) + self._reference(adapter, root) + before = rulespec_tree_digest(root) + link.unlink() + link.symlink_to(root / _OTHER_MODULE) + assert rulespec_tree_digest(root) == before + with pytest.raises(ValueError) as refused: + adapter.materialize(_fake_frame(), ["family_assistance"], 2026) + assert str(refused.value) == ( + f"Module {link} resolves to " + f"{(root / _OTHER_MODULE).resolve()}, not the file " + f"{(root / _MODULE).resolve()} this adapter's engine_ref named; " + "construct a new adapter and reference." + ) + assert programs == {} + + def test_a_module_link_repointed_after_the_reference_blocks_a_second_reference( + self, tmp_path, monkeypatch + ) -> None: + adapter, root, link = self._linked_adapter(tmp_path) + _fake_engine(adapter, monkeypatch) + self._reference(adapter, root) + adapter.materialize(_fake_frame(), ["family_assistance"], 2026) + link.unlink() + link.symlink_to(root / _OTHER_MODULE) + with pytest.raises(ValueError, match="not the file .* engine_ref named"): + self._reference(adapter, root) + + @settings( + max_examples=60, deadline=None, suppress_health_check=[HealthCheck.too_slow] + ) + @given(files=rulespec_trees(), data=st.data()) + def test_property_it_compiles_only_from_the_referenced_bytes( + self, tmp_path_factory, files, data + ) -> None: + """Invariant: a referenced adapter compiles only from referenced bytes. + + After the reference the test makes one random change, or none: edits + one byte, adds, removes or renames a file, adds a directory or + dangling link, or repoints the module's outside link at another file. + The adapter compiles iff nothing changed. When it compiles, the root + still hashes to the pinned digest and the engine was handed a module + path that resolves to the referenced file. + """ + + base = tmp_path_factory.mktemp("pin") + # A second module is always present, for the link to be repointed at. + files = {**files, _OTHER_MODULE: b"format: rulespec/v1\n"} + root = _write_tree(base / "rulespec", files) + names = sorted(files) + others = [name for name in names if name != _MODULE] + change = data.draw( + st.sampled_from( + [ + "none", + "edit", + "add", + "remove", + "rename", + "dir link", + "dangling link", + "repoint", + ] + ), + label="change", + ) + # Repointing needs the module path to be a link outside the root. + via_link = change == "repoint" or data.draw(st.booleans(), label="via_link") + module = root / _MODULE + if via_link: + link = base / "links" / "module.yaml" + link.parent.mkdir() + link.symlink_to(module) + module = link + adapter = self._adapter_at(module, root) + with pytest.MonkeyPatch.context() as patch: + programs = _fake_engine(adapter, patch) + self._reference(adapter, root) + pinned, _ = rulespec_tree_digest(root) + if change == "edit": + name = data.draw(st.sampled_from(names), label="file") + content = bytearray(files[name]) + offset = data.draw(st.integers(0, len(content) - 1), label="offset") + content[offset] ^= data.draw(st.integers(1, 255), label="mask") + (root / name).write_bytes(bytes(content)) + elif change == "add": + (root / "added.bin").write_bytes(b"") + elif change == "remove": + (root / data.draw(st.sampled_from(names), label="file")).unlink() + elif change == "rename": + name = data.draw(st.sampled_from(names), label="file") + (root / name).rename(root / f"{name}.renamed") + elif change == "dir link": + outside = _write_tree(base / "outside", {"hidden.yaml": b"x"}) + (root / "linked").symlink_to(outside, target_is_directory=True) + elif change == "dangling link": + (root / "dangling.yaml").symlink_to(base / "nowhere.yaml") + elif change == "repoint": + link.unlink() + link.symlink_to(root / data.draw(st.sampled_from(others), label="file")) + try: + adapter.materialize(_fake_frame(), ["family_assistance"], 2026) + except ValueError: + compiled = False + else: + compiled = True + assert compiled == (change == "none") + if compiled: + assert rulespec_tree_digest(root)[0] == pinned + assert {program.module_path.resolve() for program in programs.values()} == { + (root / _MODULE).resolve() + } + else: + assert programs == {} + def test_an_adapter_without_a_reference_never_hashes_its_root( self, monkeypatch ) -> None: @@ -1324,6 +1491,64 @@ def refuse(root): adapter.materialize(_fake_frame(), ["family_assistance"], 2026) +#: Variables git lists as local to one repository, each set by the property +#: test to name a clean copy (``GIT_CONFIG_COUNT`` carries core.worktree). +_REDIRECTING_VARIABLES = ( + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_COMMON_DIR", + "GIT_CONFIG_COUNT", +) + +#: Changes that leave a committed checkout short of exactly its commit. +_CHECKOUT_CHANGES = ( + "none", + "edit", + "untracked file", + "ignored file", + "staged edit, reverted on disk", + "skip-worktree", + "assume-unchanged edit", + "submodule", + "HEAD moved on", + "another commit declared", + "replace ref", + "stat-cache edit", + "clean-filter edit", + "fsmonitor-hidden edit", + "nested .git file", + "symbolic link as a plain file", + "executable mode hidden by core.filemode", + "core.worktree elsewhere", +) + + +def _aim_git_at( + patch: pytest.MonkeyPatch, repository: Path, names: Iterable[str] +) -> None: + """Set each named repository-selecting variable to point at ``repository``. + + ``GIT_CONFIG_COUNT`` carries ``-c core.worktree=``. + """ + + values = { + "GIT_DIR": repository / ".git", + "GIT_WORK_TREE": repository, + "GIT_INDEX_FILE": repository / ".git" / "index", + "GIT_OBJECT_DIRECTORY": repository / ".git" / "objects", + "GIT_COMMON_DIR": repository / ".git", + } + for name in names: + if name == "GIT_CONFIG_COUNT": + patch.setenv("GIT_CONFIG_COUNT", "1") + patch.setenv("GIT_CONFIG_KEY_0", "core.worktree") + patch.setenv("GIT_CONFIG_VALUE_0", str(repository)) + else: + patch.setenv(name, str(values[name])) + + class TestAxiomEngineRefGitCheckout: @pytest.fixture def checkout(self, tmp_path) -> tuple[Path, str]: @@ -1384,8 +1609,14 @@ def test_an_edit_hidden_by_an_index_flag_is_refused(self, checkout, flag) -> Non _git(root, "update-index", flag, _MODULE) (root / _MODULE).write_bytes(b"format: rulespec/v1\nrules: [hidden]\n") assert _git(root, "status", "--porcelain", "--ignored") == "" - with pytest.raises(ValueError, match=f"{_MODULE} is flagged"): + with pytest.raises(ValueError) as refused: self._checkout_ref(root, head) + assert str(refused.value) == ( + f"RuleSpec checkout {root.resolve()}: {_MODULE} is flagged " + "skip-worktree or assume-unchanged (1 flagged file(s)), so git status " + "does not report edits to it; clear the flags or reference a git " + "archive export instead." + ) @pytest.mark.parametrize( ("flag", "unflag"), @@ -1427,6 +1658,509 @@ def test_a_checkout_at_another_commit_is_refused(self, checkout) -> None: with pytest.raises(ValueError, match="not the declared rulespec_commit"): self._checkout_ref(root, other) + # -- submodules ------------------------------------------------------ + + def test_a_clean_submodule_is_refused(self, checkout) -> None: + # The commit records the submodule's commit, not its files. + root, _ = checkout + head, _ = _commit_submodule(root) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError) as refused: + self._checkout_ref(root, head) + assert str(refused.value) == ( + f"RuleSpec checkout {root.resolve()} contains the submodule zz/vendor " + f"(1 submodule(s)); commit {head} records a submodule's commit, not " + "its files, and git status does not report every change inside one. " + "Reference a git archive export instead." + ) + + @pytest.mark.parametrize( + "change", ["edit under ignore=all", "assume-unchanged edit", "ignored file"] + ) + def test_a_change_inside_a_submodule_that_git_status_misses_is_refused( + self, checkout, change + ) -> None: + # The review's A1, D1 and E1: git status reports nothing for each, so + # the checkout used to pass with rulespec_commit equal to HEAD. + root, _ = checkout + head, vendor = _commit_submodule( + root, ignore="all" if change == "edit under ignore=all" else None + ) + if change == "edit under ignore=all": + (vendor / "vendor.yaml").write_bytes(b"vendor: edited\n") + elif change == "assume-unchanged edit": + _git(vendor, "update-index", "--assume-unchanged", "vendor.yaml") + (vendor / "vendor.yaml").write_bytes(b"vendor: hidden\n") + else: + (vendor / "build.tmp").write_bytes(b"x") + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError, match="contains the submodule zz/vendor"): + self._checkout_ref(root, head) + + # -- the caller's git environment -------------------------------------- + + @pytest.mark.parametrize( + "variables", + [ + ("GIT_DIR", "GIT_WORK_TREE"), + ("GIT_WORK_TREE",), + ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY"), + ], + ids="+".join, + ) + def test_an_inherited_git_environment_cannot_vouch_for_a_dirty_root( + self, checkout, tmp_path, monkeypatch, variables + ) -> None: + # The variables name a clean copy at the same commit while the root + # itself is edited; git must examine the root. + root, head = checkout + decoy = tmp_path / "decoy" + shutil.copytree(root, decoy, symlinks=True) + (root / _MODULE).write_bytes(b"format: rulespec/v1\nrules: [dirty]\n") + _aim_git_at(monkeypatch, decoy, variables) + with pytest.raises(ValueError, match="modified, untracked, or ignored"): + self._checkout_ref(root, head) + + def test_an_inherited_git_environment_cannot_vouch_for_another_head( + self, checkout, tmp_path, monkeypatch + ) -> None: + # The root holds the declared commit's bytes, but its HEAD has moved + # on; a copy still at the declared commit must not answer for it. + root, head = checkout + decoy = tmp_path / "decoy" + shutil.copytree(root, decoy, symlinks=True) + _git(root, "commit", "-q", "--allow-empty", "-m", "moved") + moved = _git(root, "rev-parse", "HEAD") + monkeypatch.setenv("GIT_DIR", str(decoy / ".git")) + monkeypatch.setenv("GIT_WORK_TREE", str(decoy)) + with pytest.raises( + ValueError, match=f"is at {moved}, not the declared rulespec_commit {head}" + ): + self._checkout_ref(root, head) + + def test_a_clean_root_is_accepted_whatever_the_inherited_git_environment( + self, checkout, tmp_path, monkeypatch + ) -> None: + root, head = checkout + other = _write_tree(tmp_path / "other", {"other.yaml": b"other\n"}) + _git(other, "init", "-q") + _git(other, "add", "-A") + _git(other, "commit", "-q", "-m", "other") + _aim_git_at(monkeypatch, other, _REDIRECTING_VARIABLES) + assert json.loads(self._checkout_ref(root, head))["rulespec_commit"] == head + + def test_the_root_must_be_its_repositorys_top_level( + self, checkout, tmp_path + ) -> None: + # core.worktree points the repository at a clean copy while the root + # itself is edited, so git status examines the copy. + root, head = checkout + elsewhere = tmp_path / "elsewhere" + shutil.copytree(root, elsewhere, ignore=shutil.ignore_patterns(".git")) + _git(root, "config", "core.worktree", str(elsewhere)) + (root / _MODULE).write_bytes(b"format: rulespec/v1\nrules: [dirty]\n") + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError) as refused: + self._checkout_ref(root, head) + assert str(refused.value) == ( + f"RuleSpec checkout {root.resolve()} is not the top level of its git " + f"repository, whose work tree is {elsewhere.resolve()}; reference a " + "git archive export instead." + ) + + def test_a_replace_ref_cannot_stand_in_for_the_declared_commit( + self, checkout + ) -> None: + # refs/replace makes the declared commit read as another whose tree + # the root holds; HEAD still names the declared commit. + root, head = checkout + (root / _MODULE).write_bytes(b"format: rulespec/v1\nrules: [other]\n") + _git(root, "commit", "-q", "-a", "-m", "other") + other = _git(root, "rev-parse", "HEAD") + _git(root, "replace", head, other) + _git(root, "update-ref", "HEAD", head) + assert _git(root, "rev-parse", "HEAD") == head + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError, match="modified, untracked, or ignored"): + self._checkout_ref(root, head) + + # -- bytes git status does not see ---------------------------------------- + + def test_a_same_size_edit_the_stat_cache_hides_is_refused(self, tmp_path) -> None: + # With core.trustctime=false git trusts size and mtime; an in-place + # edit of the same size with its mtime put back is invisible to it. + root, head = _old_mtime_checkout(tmp_path / "stat-cache") + _hide_a_same_size_edit_from_the_stat_cache(root) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError) as refused: + self._checkout_ref(root, head) + assert str(refused.value).startswith( + f"RuleSpec checkout {root.resolve()}: {_MODULE} is not the blob " + ) + + def test_a_same_size_edit_a_clean_filter_hides_is_refused(self, checkout) -> None: + # A clean filter maps the edited bytes back to the committed blob, so + # git status sees no change. + root, head = checkout + _hide_a_same_size_edit_behind_a_clean_filter(root) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError, match=f"{_MODULE} is not the blob"): + self._checkout_ref(root, head) + + @pytest.mark.parametrize("hidden", ["edit", "deletion"]) + def test_a_change_a_quiet_file_system_monitor_hides_is_refused( + self, checkout, tmp_path, hidden + ) -> None: + # git status trusts core.fsmonitor for which tracked files changed; a + # monitor that reports nothing hides an edit or a deletion. + root, head = checkout + _quiet_file_system_monitor(root, tmp_path / "quiet-fsmonitor") + if hidden == "edit": + (root / _MODULE).write_bytes(b"format: rulespec/v1\nrules: [quiet]\n") + expected = f"{_MODULE} is not the blob" + else: + (root / "README.md").unlink() + expected = "lacks README.md, unlike commit" + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError, match=expected): + self._checkout_ref(root, head) + + @pytest.mark.parametrize("entry", ["file", "directory"]) + def test_a_nested_dot_git_entry_git_status_skips_is_refused( + self, checkout, entry + ) -> None: + # git's untracked scan skips every entry named .git, so bytes the + # digest hashes, and the commit does not hold, pass git status. + root, head = checkout + _plant_nested_dot_git(root, entry) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError, match="holds zz/policies/.git"): + self._checkout_ref(root, head) + + def test_a_symbolic_link_checked_out_as_a_plain_file_is_refused( + self, checkout + ) -> None: + # Under core.symlinks=false git writes a committed link as a plain + # file holding the link's blob, which git status calls unchanged. The + # bytes match the blob; the mode does not. + root, _ = checkout + head = _commit_symlink_as_a_plain_file(root) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError) as refused: + self._checkout_ref(root, head) + assert str(refused.value) == ( + f"RuleSpec checkout {root.resolve()}: commit {head} records " + f"{_ALIAS} with mode 120000, not as a regular file, though the " + "checkout holds a plain file there; reference a git archive export " + "instead." + ) + + @pytest.mark.parametrize( + ("committed_mode", "changed_mode"), [(0o644, 0o755), (0o755, 0o644)] + ) + def test_an_executable_mode_change_git_status_hides_is_refused( + self, checkout, committed_mode, changed_mode + ) -> None: + root, head = checkout + module = root / _MODULE + if committed_mode == 0o755: + module.chmod(committed_mode) + _git(root, "update-index", "--chmod=+x", _MODULE) + _git(root, "commit", "-q", "-m", "executable module") + head = _git(root, "rev-parse", "HEAD") + _hide_an_executable_mode_change(root, changed_mode) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError) as refused: + self._checkout_ref(root, head) + expected_mode = "100755" if committed_mode == 0o755 else "100644" + actual_mode = "100755" if changed_mode == 0o755 else "100644" + assert str(refused.value) == ( + f"RuleSpec checkout {root.resolve()}: {_MODULE} has mode " + f"{actual_mode}, not the mode {expected_mode} commit {head} records " + "for it, though git status reports no change; reference a git " + "archive export instead." + ) + + def test_a_sha256_repository_is_accepted_and_checked_byte_for_byte( + self, tmp_path + ) -> None: + root = _write_tree(tmp_path / "rulespec-sha256", _BASE_TREE) + _git(root, "init", "-q", "--object-format=sha256") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", "fixture") + head = _git(root, "rev-parse", "HEAD") + assert len(head) == 64 + assert json.loads(self._checkout_ref(root, head))["rulespec_commit"] == head + _hide_a_same_size_edit_behind_a_clean_filter(root) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError, match=f"{_MODULE} is not the blob"): + self._checkout_ref(root, head) + + # -- the invariant ------------------------------------------------------- + + @settings( + max_examples=50, deadline=None, suppress_health_check=[HealthCheck.too_slow] + ) + @given( + change=st.sampled_from(_CHECKOUT_CHANGES), + variables=st.sets(st.sampled_from(_REDIRECTING_VARIABLES)), + ) + def test_property_accepted_iff_the_root_holds_exactly_the_commit( + self, tmp_path_factory, git_template, change, variables + ) -> None: + """Invariant: a git root is accepted iff it is exactly the declared commit. + + The root is a copy of a committed checkout. The test makes one change + that leaves it short of exactly the declared commit, or none, and sets + any subset of git's repository-selecting variables to a clean copy + at the same commit. The root is accepted iff nothing changed, and + then the reference records its own HEAD. + """ + + template, decoy, head = git_template + root = tmp_path_factory.mktemp("checkout") / "rulespec-git" + shutil.copytree(template, root, symlinks=True) + declared = _apply_checkout_change(root, change, head, tmp_path_factory) + with pytest.MonkeyPatch.context() as patch: + _aim_git_at(patch, decoy, variables) + try: + reference = self._checkout_ref(root, declared) + except ValueError: + reference = None + assert (reference is not None) == (change == "none") + if reference is not None: + assert json.loads(reference)["rulespec_commit"] == head + + +class TestGitBlobId: + @settings(max_examples=60, deadline=None) + @given(content=st.binary(max_size=512)) + def test_differential_blob_ids_equal_git_hash_object( + self, blob_repositories, content + ) -> None: + """The byte check's blob ids are the ones git computes, in both formats.""" + + for algorithm, repository in blob_repositories.items(): + expected = ( + subprocess.run( + [ + "git", + "-C", + str(repository), + "hash-object", + "--no-filters", + "--stdin", + ], + input=content, + capture_output=True, + check=True, + env=_GIT_ENV, + ) + .stdout.decode("ascii") + .strip() + ) + assert axiom_adapter._git_blob_id(content, algorithm) == expected + + +@pytest.fixture(scope="module") +def blob_repositories(tmp_path_factory) -> dict[str, Path]: + repositories = {} + for algorithm in ("sha1", "sha256"): + repository = tmp_path_factory.mktemp(f"blob-{algorithm}") + _git(repository, "init", "-q", f"--object-format={algorithm}") + assert _git(repository, "rev-parse", "--show-object-format") == algorithm + repositories[algorithm] = repository + return repositories + + +@pytest.fixture(scope="module") +def git_template(tmp_path_factory) -> tuple[Path, Path, str]: + """A committed checkout to copy, a clean decoy copy of it, and its commit. + + Every file carries an old mtime, so an in-place edit can restore it and + leave the stat cache matching (the ``stat-cache edit`` change). + """ + + root, head = _old_mtime_checkout( + tmp_path_factory.mktemp("template") / "rulespec-git" + ) + decoy = tmp_path_factory.mktemp("decoy") / "rulespec-git" + shutil.copytree(root, decoy, symlinks=True) + return root, decoy, head + + +def _commit_submodule(root: Path, *, ignore: str | None = None) -> tuple[str, Path]: + """Commit an embedded repository at ``zz/vendor`` as a submodule of ``root``.""" + + vendor = _write_tree( + root / "zz/vendor", {"vendor.yaml": b"vendor: 1\n", ".gitignore": b"*.tmp\n"} + ) + _git(vendor, "init", "-q") + _git(vendor, "add", "-A") + _git(vendor, "commit", "-q", "-m", "vendor") + _git(root, "add", "zz/vendor") + if ignore is not None: + (root / ".gitmodules").write_text( + '[submodule "zz/vendor"]\n\tpath = zz/vendor\n\turl = ./zz/vendor\n' + f"\tignore = {ignore}\n", + encoding="utf-8", + ) + _git(root, "add", ".gitmodules") + _git(root, "commit", "-q", "-m", "vendor") + return _git(root, "rev-parse", "HEAD"), vendor + + +_OLD_MTIME = 1_600_000_000 + + +def _old_mtime_checkout(root: Path) -> tuple[Path, str]: + """A committed checkout whose files all carry an old mtime.""" + + _write_tree(root, {**_BASE_TREE, ".gitignore": b"*.local.yaml\n"}) + for path in root.rglob("*"): + if path.is_file(): + os.utime(path, (_OLD_MTIME, _OLD_MTIME)) + _git(root, "init", "-q") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", "fixture") + return root, _git(root, "rev-parse", "HEAD") + + +def _hide_a_same_size_edit_from_the_stat_cache(root: Path) -> None: + """Edit the module in place, same size, and put its old mtime back. + + With core.trustctime=false git's stat check ignores the ctime the edit + moves, so it sees size and mtime unchanged. + """ + + _git(root, "config", "core.trustctime", "false") + # Record this copy's own inode and ctime in the index first. + _git(root, "update-index", "--refresh") + module = root / _MODULE + content = module.read_bytes() + edited = content.replace(b"rules: []", b"rules: [x") + assert len(edited) == len(content) and edited != content + module.write_bytes(edited) + os.utime(module, (_OLD_MTIME, _OLD_MTIME)) + + +def _quiet_file_system_monitor(root: Path, hook: Path) -> None: + """Install a core.fsmonitor hook that never reports a change. + + The index records the monitor and marks every entry valid, so git status + then takes the hook's word that nothing changed. + """ + + hook.write_text("#!/bin/sh\nprintf 'quiet\\0'\n", encoding="utf-8") + hook.chmod(0o755) + _git(root, "config", "core.fsmonitor", str(hook)) + _git(root, "update-index", "--fsmonitor") + _git(root, "status") + _git(root, "status") + + +def _plant_nested_dot_git(root: Path, entry: str) -> None: + """Add a ``.git`` file or directory below the root that is no repository.""" + + if entry == "file": + (root / "zz/policies/.git").write_bytes(b"gitdir: nowhere\n") + else: + _write_tree(root / "zz/policies/.git", {"notes.txt": b"not a repository\n"}) + + +_ALIAS = "zz/policies/tests/alias.yaml" + + +def _commit_symlink_as_a_plain_file(root: Path) -> str: + """Commit a symbolic link, then hold it as a plain file under core.symlinks=false.""" + + alias = root / _ALIAS + alias.symlink_to(Path(_MODULE).name) + _git(root, "add", _ALIAS) + _git(root, "commit", "-q", "-m", "alias") + _git(root, "config", "core.symlinks", "false") + alias.unlink() + alias.write_bytes(Path(_MODULE).name.encode("utf-8")) + _git(root, "update-index", "--refresh") + assert _git(root, "ls-files", "--stage", _ALIAS).startswith("120000 ") + return _git(root, "rev-parse", "HEAD") + + +def _hide_a_same_size_edit_behind_a_clean_filter(root: Path) -> None: + """Edit the module, same size, behind a clean filter that undoes the edit.""" + + (root / ".git" / "info").mkdir(exist_ok=True) + (root / ".git" / "info" / "attributes").write_text( + "*.yaml filter=restore\n", encoding="utf-8" + ) + _git(root, "config", "filter.restore.clean", "sed s/HIDDEN/rules:/") + module = root / _MODULE + content = module.read_bytes() + edited = content.replace(b"rules:", b"HIDDEN") + assert len(edited) == len(content) and edited != content + module.write_bytes(edited) + + +def _hide_an_executable_mode_change(root: Path, mode: int = 0o755) -> None: + """Change the module's executable bit while git ignores worktree modes.""" + + _git(root, "config", "core.filemode", "false") + (root / _MODULE).chmod(mode) + + +def _apply_checkout_change(root: Path, change: str, head: str, tmp_path_factory) -> str: + """Apply one ``_CHECKOUT_CHANGES`` entry to ``root``; return the commit to declare.""" + + module = root / _MODULE + if change == "edit": + module.write_bytes(b"format: rulespec/v1\nrules: [edited]\n") + elif change == "untracked file": + (root / "zz/untracked.yaml").write_bytes(b"x") + elif change == "ignored file": + (root / "zz/override.local.yaml").write_bytes(b"x") + elif change == "staged edit, reverted on disk": + content = module.read_bytes() + module.write_bytes(b"format: rulespec/v1\nrules: [staged]\n") + _git(root, "add", _MODULE) + module.write_bytes(content) + elif change == "skip-worktree": + _git(root, "update-index", "--skip-worktree", _MODULE) + elif change == "assume-unchanged edit": + _git(root, "update-index", "--assume-unchanged", _MODULE) + module.write_bytes(b"format: rulespec/v1\nrules: [hidden]\n") + elif change == "submodule": + head, _ = _commit_submodule(root) + elif change == "HEAD moved on": + _git(root, "commit", "-q", "--allow-empty", "-m", "moved") + elif change == "another commit declared": + return "1" * len(head) if head != "1" * len(head) else "2" * len(head) + elif change == "replace ref": + module.write_bytes(b"format: rulespec/v1\nrules: [other]\n") + _git(root, "commit", "-q", "-a", "-m", "other") + _git(root, "replace", head, _git(root, "rev-parse", "HEAD")) + _git(root, "update-ref", "HEAD", head) + elif change == "stat-cache edit": + _hide_a_same_size_edit_from_the_stat_cache(root) + elif change == "clean-filter edit": + _hide_a_same_size_edit_behind_a_clean_filter(root) + elif change == "fsmonitor-hidden edit": + _quiet_file_system_monitor(root, root.parent / "quiet-fsmonitor") + module.write_bytes(b"format: rulespec/v1\nrules: [quiet]\n") + elif change == "nested .git file": + _plant_nested_dot_git(root, "file") + elif change == "symbolic link as a plain file": + head = _commit_symlink_as_a_plain_file(root) + elif change == "executable mode hidden by core.filemode": + _hide_an_executable_mode_change(root) + elif change == "core.worktree elsewhere": + elsewhere = tmp_path_factory.mktemp("elsewhere") / "rulespec-git" + shutil.copytree(root, elsewhere, ignore=shutil.ignore_patterns(".git")) + _git(root, "config", "core.worktree", str(elsewhere)) + module.write_bytes(b"format: rulespec/v1\nrules: [dirty]\n") + else: + assert change == "none", change + return head + # ---------------------------------------------------------------------- # Real engine (skipped without axiom_rules_engine) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py b/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py index 7a75d2ec89..3bc855ac3b 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_rules_kernels.py @@ -8,8 +8,10 @@ both engines, and against two single-engine ``simulate.rules@1`` graph runs. """ +import importlib.util import itertools import json +import sys from collections.abc import Mapping, Sequence from pathlib import Path @@ -193,6 +195,88 @@ def _by_ref(**extra) -> SimulateRulesByRefKernel: ) +def _fixture_axiom_engine() -> AxiomEngine: + return AxiomEngine( + _RULESPEC_ROOT / "zz/policies/tests/axiom_toy_family.yaml", + schema=NZ_SCHEMA, + rulespec_roots=(_RULESPEC_ROOT,), + ) + + +def _module_order(classes) -> tuple[type, ...]: + """Adapter classes in the order the kernel hashes them.""" + + return tuple(sorted(classes, key=lambda cls: (cls.__module__, cls.__qualname__))) + + +def _hash_of(classes) -> str: + """The by-ref kernel's implementation hash over ``classes`` in this order.""" + + return source_hash( + SimulateRulesByRefKernel, + SimulateRulesKernel, + *classes, + frame_bundle_module, + frame_rules_module, + frame_schema_module, + ) + + +# Two importable modules, each defining a rules engine named ``TwinEngine``. +# Their names sort before ``microcosm.frame.adapters.axiom``. +_TWIN_MODULES = ("kernel_hash_probe_a", "kernel_hash_probe_b") +_TWIN_SOURCE = """\ +# A rules engine outside the test module, for implementation-hash tests. + +TAG = {tag!r} + + +class TwinEngine: + def variable_metadata(self, name): + raise ValueError(name) + + def variables(self): + return () + + def entity_schema(self): + return None + + def materialize(self, bundle, variables, period): + return {{}} + + def export_contract(self): + return None + + def write_dataset(self, bundle, path, period): + return None +""" + + +@pytest.fixture(scope="module") +def twin_engine_classes(tmp_path_factory) -> tuple[type, type]: + """``TwinEngine`` from each of two modules written for this test module. + + ``source_hash`` locates a class's module through ``sys.modules``, so the + modules are registered there while the tests run. + """ + + directory = tmp_path_factory.mktemp("kernel-hash-probes") + classes = [] + try: + for name in _TWIN_MODULES: + path = directory / f"{name}.py" + path.write_text(_TWIN_SOURCE.format(tag=name), encoding="utf-8") + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + classes.append(module.TwinEngine) + yield tuple(classes) + finally: + for name in _TWIN_MODULES: + sys.modules.pop(name, None) + + @st.composite def nz_frames(draw: st.DrawFn) -> Frame: """A nested NZ frame with person and family inputs and household weights.""" @@ -347,8 +431,8 @@ def test_the_hash_is_independent_of_binding_order(self) -> None: Three adapter classes from three modules make the source order observable: all six binding orders must give the hash of the classes in ``(module, qualname)`` order. These three classes sort the same way - by qualname alone, so the test pins order independence, not the - choice of sort key. + by qualname alone, so this test pins order independence; the two + tests after it pin the sort key. """ bindings = { @@ -379,6 +463,119 @@ def test_the_hash_is_independent_of_binding_order(self) -> None: kernel = SimulateRulesByRefKernel({ref: bindings[ref] for ref in order}) assert kernel.implementation_hash() == expected, order + def test_the_hash_is_independent_of_ref_names(self) -> None: + """Invariant: which reference names which class never moves the hash. + + Every assignment of three reference names to three adapter classes + from three modules, bound in every order, gives the hash of the + classes in ``(module, qualname)`` order. Some assignments put the + classes in another module order when sorted by reference name, so a + kernel that ordered its classes that way fails here. + """ + + engines = (_fixture_axiom_engine(), _StubRulesEngine(), _PersonEngine()) + expected = _hash_of(_module_order({type(engine) for engine in engines})) + names = ("axiom", "person", "stub") + by_ref_name = set() + for assignment in itertools.permutations(engines): + bindings = dict(zip(names, assignment, strict=True)) + by_ref_name.add( + _hash_of(dict.fromkeys(type(bindings[ref]) for ref in sorted(bindings))) + ) + for order in itertools.permutations(names): + kernel = SimulateRulesByRefKernel({ref: bindings[ref] for ref in order}) + assert kernel.implementation_hash() == expected, (assignment, order) + # The assignments discriminate: ordering by reference name would move + # the hash for at least one of them. + assert by_ref_name - {expected} + + def test_classes_sharing_a_qualname_are_ordered_by_module( + self, twin_engine_classes + ) -> None: + """Invariant: the module, not the qualname alone, orders the classes. + + Two classes named ``TwinEngine`` in two modules that sort before + ``microcosm.frame.adapters.axiom`` are bound with ``AxiomEngine``. By + qualname alone ``AxiomEngine`` would come first; by module it comes + last, and only that order gives the expected hash. + """ + + first, second = twin_engine_classes + assert first.__qualname__ == second.__qualname__ + assert first.__module__ != second.__module__ + engines = { + "twin-a": first(), + "twin-b": second(), + "axiom": _fixture_axiom_engine(), + } + classes = _module_order({type(engine) for engine in engines.values()}) + assert classes == (first, second, AxiomEngine) + expected = _hash_of(classes) + by_qualname = sorted(classes, key=lambda cls: cls.__qualname__) + assert by_qualname[0] is AxiomEngine + assert _hash_of(by_qualname) != expected + for order in itertools.permutations(engines): + for names in itertools.permutations(engines): + instances = [engines[ref] for ref in order] + kernel = SimulateRulesByRefKernel( + dict(zip(names, instances, strict=True)) + ) + assert kernel.implementation_hash() == expected, (order, names) + # Same qualname, different module: each binds its own source. + assert ( + SimulateRulesByRefKernel({"twin": first()}).implementation_hash() + != SimulateRulesByRefKernel({"twin": second()}).implementation_hash() + ) + + @settings(max_examples=120, deadline=None) + @given(data=st.data()) + def test_property_the_hash_depends_only_on_the_bound_classes( + self, twin_engine_classes, data + ) -> None: + """Invariant: the hash is a function of the set of adapter classes. + + Any multiset of bindings over six adapter classes from five modules, + under any distinct reference names and in any order, hashes as the + set of its classes in ``(module, qualname)`` order. Two binding sets + hash alike exactly when their classes come from the same modules + (``source_hash`` hashes each module file once). + """ + + factories = ( + _fixture_axiom_engine, + _StubRulesEngine, + _PersonEngine, + _FamilyEngine, + *twin_engine_classes, + ) + names = st.text( + alphabet="abcdefghijklmnopqrstuvwxyz-_.{}:0123456789", + min_size=1, + max_size=12, + ) + + def draw_kernel() -> tuple[SimulateRulesByRefKernel, frozenset[str]]: + chosen = data.draw( + st.lists(st.sampled_from(factories), min_size=1, max_size=6) + ) + refs = data.draw( + st.lists(names, min_size=len(chosen), max_size=len(chosen), unique=True) + ) + bindings = { + ref: factory() for ref, factory in zip(refs, chosen, strict=True) + } + order = data.draw(st.permutations(list(bindings))) + kernel = SimulateRulesByRefKernel({ref: bindings[ref] for ref in order}) + classes = {type(engine) for engine in bindings.values()} + assert kernel.implementation_hash() == _hash_of(_module_order(classes)) + return kernel, frozenset(cls.__module__ for cls in classes) + + first, first_modules = draw_kernel() + second, second_modules = draw_kernel() + assert (first.implementation_hash() == second.implementation_hash()) == ( + first_modules == second_modules + ) + def test_binding_a_new_adapter_class_moves_the_hash(self) -> None: # Documented: a new class's source joins the hash. assert ( From b06217f065aafdad7cb19c3eeea75e75a208d2af Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 19:22:06 -0400 Subject: [PATCH 2/2] Pair Unicode-equivalent checkout paths one to one On macOS with core.precomposeunicode=true, git records a path in NFC while the file system lists the NFD name the file was created under. The checkout check compared the two path sets as strings and refused a clean checkout ("2 path(s) differ"). Committed paths now pair with files one to one: identical names first, then the single canonically equivalent entry that the committed path opens (Path.samefile). Ambiguous or unmatched names are refused; the byte, executable-mode and directory-digest checks are unchanged, and nothing is normalized wholesale. Regression tests build the clean macOS case under tmp_path and show that a hidden edit or mode change to the paired file is still refused. A commit holding both spellings of one name is refused where one file opens under both. A property test checks the pairing against a brute-force matching on the real file system. The changelog and docstrings now state that the exact-commit check assumes an intact object database. Co-Authored-By: Claude Opus 5.5 --- changelog.d/axiom-adapter-hardening.fixed.md | 10 +- .../src/microcosm/frame/adapters/axiom.py | 85 ++++- .../shared/test_axiom_graph_adapter.py | 323 +++++++++++++++++- 3 files changed, 399 insertions(+), 19 deletions(-) diff --git a/changelog.d/axiom-adapter-hardening.fixed.md b/changelog.d/axiom-adapter-hardening.fixed.md index 5dd9229b54..e5d28c3ed1 100644 --- a/changelog.d/axiom-adapter-hardening.fixed.md +++ b/changelog.d/axiom-adapter-hardening.fixed.md @@ -10,6 +10,14 @@ executable bit, so a change that trusted under `core.trustctime=false`, by a clean filter or by a file system monitor, a nested `.git` entry, or a symbolic link held as a plain file under `core.symlinks=false`, or an executable-bit change hidden by -`core.filemode=false`. A referenced adapter also records the file its module +`core.filemode=false`. Committed paths pair with files one to one: a name git +records precomposed (usually Unicode NFC) matches the file listed under an +equivalent NFD name only when that one file opens under both, as on macOS, and +one directory entry never answers for two committed names. The commit's paths and blob names are read +from the repository's object database, whose commit and tree objects are not +rehashed, so this check assumes the database is intact: deliberate +object-store tampering, such as a nested tree rewritten under its old object +name, is out of its scope, though the directory digest still pins the bytes +present. A referenced adapter also records the file its module path resolves to, and refuses to compile or take a second reference once the path resolves to another file. diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py index 3b9f3b5ee6..b3afe7c520 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom.py @@ -98,7 +98,8 @@ import os import re import subprocess -from collections.abc import Mapping, Sequence +import unicodedata +from collections.abc import Collection, Mapping, Sequence from dataclasses import dataclass from datetime import date from functools import cache @@ -1185,10 +1186,20 @@ def axiom_engine_ref( tracked file flagged skip-worktree or assume-unchanged (``git status`` does not report edits to those), and holds outside ``.git`` exactly the commit's files, each byte for byte the blob the commit records and with - its executable bit. Git runs + its executable bit. A recorded name matches a file whose name is the + same or, when that one file opens under both, canonically equivalent + (git on macOS with ``core.precomposeunicode=true`` records precomposed + names, usually NFC, that the file system may list in NFD); one directory + entry never answers for two recorded names. Git runs without the caller's repository-selecting variables (``GIT_DIR``, ``GIT_WORK_TREE`` and the rest of ``git rev-parse --local-env-vars``) and without replace refs, so neither can point the check at other objects. + The commit's file list and blob names are read through git from the + repository's object database, whose commit and tree objects are not + rehashed, so the check assumes that database is intact: it does not + detect an object rewritten under another object's name (a substituted + nested tree, say), though the directory digest still pins the bytes + present. Its ``.git`` is then hashed like any other file, as the graph's source key hashes it. That makes the reference specific to one clone: two clones or worktrees of one commit @@ -1565,7 +1576,12 @@ def _require_clean_checkout(root: Path, commit: str) -> None: ``core.trustctime=false``, by a clean filter, or by a file system monitor; a nested ``.git`` entry, which git's untracked scan skips; or a committed symbolic link held as a plain file under ``core.symlinks=false``; or an - executable-bit change hidden by ``core.filemode=false``. + executable-bit change hidden by ``core.filemode=false``. Recorded paths + pair with files one to one as :func:`_match_checkout_paths` describes, + so an NFC name git records matches the one file the file system lists + under an equivalent NFD name. The paths and blob names come from + ``git ls-tree``, which reads the object database; the commit and tree + objects are not rehashed, so an intact object database is assumed. Raises: ValueError: If git fails or times out; the root is not its @@ -1675,11 +1691,11 @@ def git(*args: str) -> str: fields, path = entry.split("\t", 1) mode, _, name = fields.split(" ") recorded[path] = (mode, name) - present = _checkout_files(root) - unexpected = sorted(present.keys() ^ recorded.keys()) + files, lacking, extra = _match_checkout_paths(root, recorded, _checkout_files(root)) + unexpected = sorted((*lacking, *extra)) if unexpected: first = unexpected[0] - state = "holds" if first in present else "lacks" + state = "lacks" if first in lacking else "holds" raise ValueError( f"RuleSpec checkout {root} {state} {first}, unlike commit {commit} " f"({len(unexpected)} path(s) differ), though git status reports no " @@ -1694,7 +1710,7 @@ def git(*args: str) -> str: f"mode {mode}, not as a regular file, though the checkout holds " "a plain file there; reference a git archive export instead." ) - file = present[path] + file = files[path] if not file.is_file() or _git_blob_id(file.read_bytes(), algorithm) != name: raise ValueError( f"RuleSpec checkout {root}: {path} is not the blob {name} commit " @@ -1764,6 +1780,61 @@ def _checkout_files(root: Path) -> dict[str, Path]: return present +def _match_checkout_paths( + root: Path, recorded: Collection[str], present: Mapping[str, Path] +) -> tuple[dict[str, Path], list[str], list[str]]: + """Pair each path a commit records with the checkout entry holding it, one to one. + + Git and the file system can spell one file's name differently: on macOS + with ``core.precomposeunicode=true``, git records a name in Unicode NFC + while the file system lists it as it was created, for instance in NFD, + and both spellings open the same file. A recorded path pairs with the + entry of the identical name when there is one. Otherwise it pairs with + the one remaining entry whose name is canonically equivalent (the same + NFD form), and only when ``root / path`` opens that entry's file + (:meth:`Path.samefile`). Each entry pairs at most once; when a form has + more than one unpaired path or entry, none of them pairs. Nothing is + normalized wholesale: where the file system keeps an NFC and an NFD name + apart they are two entries, neither standing in for the other, and one + file cannot stand in for two recorded spellings. Names that differ in + case are not equivalent. + + Args: + root: The checkout's top level. + recorded: Paths the commit records, POSIX and relative to ``root``. + present: :func:`_checkout_files` of ``root``. + + Returns: + The entry each paired recorded path names, the recorded paths left + unpaired, and the entries left unpaired (as the file system spells + them); both lists sorted. + """ + + paired = {path: present[path] for path in recorded if path in present} + lacking: dict[str, list[str]] = {} + for path in recorded: + if path not in paired: + lacking.setdefault(unicodedata.normalize("NFD", path), []).append(path) + extra: dict[str, list[str]] = {} + for path in present: + if path not in paired: + extra.setdefault(unicodedata.normalize("NFD", path), []).append(path) + unpaired: list[str] = [] + for form, paths in lacking.items(): + entries = extra.get(form, []) + if len(paths) == 1 and len(entries) == 1: + try: + same = (root / paths[0]).samefile(present[entries[0]]) + except OSError: + same = False + if same: + paired[paths[0]] = present[entries.pop()] + continue + unpaired.extend(paths) + leftover = sorted(path for entries in extra.values() for path in entries) + return paired, sorted(unpaired), leftover + + def _git_blob_id(content: bytes, algorithm: str) -> str: """The object name git gives ``content`` as a blob (no filters applied). diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py b/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py index e2477c9055..e939327fbc 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_axiom_graph_adapter.py @@ -22,13 +22,15 @@ import shutil import subprocess import tarfile -from collections.abc import Iterable, Mapping +import unicodedata +from collections.abc import Collection, Iterable, Mapping +from itertools import permutations from pathlib import Path import numpy as np import pandas as pd import pytest -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, event, given, settings from hypothesis import strategies as st from microcosm.frame import ExportContract, Frame, WeightKind, Weights @@ -950,6 +952,8 @@ def test_only_axiom_engines_are_accepted(self) -> None: _MODULE = "zz/policies/tests/toy.yaml" _OTHER_MODULE = "zz/policies/tests/other.yaml" +#: An auxiliary RuleSpec file whose directory and name are spelled in NFD. +_DECOMPOSED = "zz/re\u0300gles/cafe\u0301.yaml" _BASE_TREE = { _MODULE: b"format: rulespec/v1\nrules: []\n", "zz/policies/shared/rates.yaml": b"format: rulespec/v1\nrules:\n - name: r\n", @@ -1896,6 +1900,113 @@ def test_a_sha256_repository_is_accepted_and_checked_byte_for_byte( with pytest.raises(ValueError, match=f"{_MODULE} is not the blob"): self._checkout_ref(root, head) + # -- names git and the file system spell differently --------------------- + + def _decomposed_checkout(self, root: Path) -> tuple[Path, str]: + """Commit a tree holding ``_DECOMPOSED``; skip unless git records it in NFC. + + Git on macOS with ``core.precomposeunicode=true`` records the NFC + spelling, while the file system lists the NFD name the file was + created under. Where git records the name as listed, the two never + diverge and the case does not arise. + """ + + _write_tree(root, {**_BASE_TREE, _DECOMPOSED: b"rules: aux\n"}) + _git(root, "init", "-q") + _git(root, "config", "core.precomposeunicode", "true") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", "decomposed names") + recorded = _git(root, "ls-tree", "-r", "-z", "--name-only", "HEAD").split("\0") + listed = axiom_adapter._checkout_files(root) + composed = unicodedata.normalize("NFC", _DECOMPOSED) + if composed not in recorded or _DECOMPOSED not in listed: + pytest.skip( + "git here records the decomposed name as the file system lists " + "it, so a commit's NFC name and an NFD file never name one file" + ) + assert composed not in listed and _DECOMPOSED not in recorded + assert (root / composed).samefile(root / _DECOMPOSED) + assert _git(root, "status", "--porcelain", "--ignored") == "" + return root, _git(root, "rev-parse", "HEAD") + + def test_a_clean_checkout_with_decomposed_names_is_accepted(self, tmp_path) -> None: + # The review of microcosm#1139: git lists the commit's NFC path, the + # file system the NFD one, and both open the same file; comparing the + # two path sets as strings refused this clean checkout. + root, head = self._decomposed_checkout(tmp_path / "rulespec-git") + reference = self._checkout_ref(root, head) + assert json.loads(reference)["rulespec_commit"] == head + assert self._checkout_ref(root, head) == reference + + @pytest.mark.parametrize("hidden", ["clean-filter edit", "executable bit"]) + def test_a_hidden_change_to_a_decomposed_file_is_refused( + self, tmp_path, hidden + ) -> None: + # A file paired through an equivalent spelling still has its bytes and + # mode checked; the error names the path as the commit records it. + root, head = self._decomposed_checkout(tmp_path / "rulespec-git") + composed = unicodedata.normalize("NFC", _DECOMPOSED) + if hidden == "clean-filter edit": + _hide_a_same_size_edit_behind_a_clean_filter(root, _DECOMPOSED) + expected = ( + f"RuleSpec checkout {root.resolve()}: {composed} is not the blob " + ) + else: + _hide_an_executable_mode_change(root, 0o755, _DECOMPOSED) + expected = ( + f"RuleSpec checkout {root.resolve()}: {composed} has mode 100755, " + "not the mode 100644 " + ) + assert _git(root, "status", "--porcelain", "--ignored") == "" + with pytest.raises(ValueError) as refused: + self._checkout_ref(root, head) + assert str(refused.value).startswith(expected) + + def test_one_file_cannot_stand_for_two_committed_spellings( + self, checkout, tmp_path + ) -> None: + # A commit can record one accented name twice, once in NFC and once in NFD, + # here with the same bytes. Where the file system opens one file under + # both names, the checkout holds one file and git status stays clean; + # pairing by normalized name, or letting one file answer for both + # paths, would accept it. Where the file system keeps the names apart, + # the checkout holds two files and each pairs with its own path. + root, _ = checkout + composed, decomposed = "caf\u00e9.yaml", "cafe\u0301.yaml" + source = tmp_path / "spelled-twice" + source.write_bytes(b"spelled: twice\n") + blob = _git(root, "hash-object", "-w", "--no-filters", str(source)) + for name in (composed, decomposed): + # Without precomposition git keeps the NFD spelling as given. + _git( + root, + "-c", + "core.precomposeunicode=false", + "update-index", + "--add", + "--cacheinfo", + f"100644,{blob},{name}", + ) + _git(root, "commit", "-q", "-m", "two spellings") + head = _git(root, "rev-parse", "HEAD") + _git(root, "reset", "-q", "--hard") + recorded = _git(root, "ls-tree", "-r", "-z", "--name-only", "HEAD").split("\0") + assert {composed, decomposed} <= set(recorded) + assert _git(root, "status", "--porcelain", "--ignored") == "" + listed = set(axiom_adapter._checkout_files(root)) & {composed, decomposed} + if listed == {composed, decomposed}: + assert not (root / composed).samefile(root / decomposed) + assert json.loads(self._checkout_ref(root, head))["rulespec_commit"] == head + return + (lost,) = {composed, decomposed} - listed + with pytest.raises(ValueError) as refused: + self._checkout_ref(root, head) + assert str(refused.value) == ( + f"RuleSpec checkout {root.resolve()} lacks {lost}, unlike commit " + f"{head} (1 path(s) differ), though git status reports no change; " + "reference a git archive export instead." + ) + # -- the invariant ------------------------------------------------------- @settings( @@ -1910,7 +2021,8 @@ def test_property_accepted_iff_the_root_holds_exactly_the_commit( ) -> None: """Invariant: a git root is accepted iff it is exactly the declared commit. - The root is a copy of a committed checkout. The test makes one change + The root is a copy of a committed checkout holding ``_DECOMPOSED``, + whose NFC record pairs with its NFD file on macOS. The test makes one change that leaves it short of exactly the declared commit, or none, and sets any subset of git's repository-selecting variables to a clean copy at the same commit. The root is accepted iff nothing changed, and @@ -1962,6 +2074,183 @@ def test_differential_blob_ids_equal_git_hash_object( assert axiom_adapter._git_blob_id(content, algorithm) == expected +#: Spellings of one name: canonically equivalent (one NFD form) within a +#: group, distinct across groups. The third spellings are the Angstrom sign +#: and combining marks out of canonical order. "Plain" and "plain" differ only +#: in case, which is not equivalence, though a case-insensitive file system +#: opens one file under both. +_NAME_SPELLINGS = ( + ("caf\u00e9", "cafe\u0301"), + ("\u00c5", "A\u030a", "\u212b"), + ("\u1e69", "s\u0323\u0307", "s\u0307\u0323"), + ("plain",), + ("Plain",), +) +_DIRECTORY_SPELLINGS = (("r\u00e8gles", "re\u0300gles"),) +_PATH_SHAPES = st.tuples( + st.sampled_from((None, *range(len(_DIRECTORY_SPELLINGS)))), + st.integers(0, len(_NAME_SPELLINGS) - 1), +) + + +def _spellings_of(shape: tuple[int | None, int]) -> list[str]: + """Every spelling of one path shape (an optional directory and a name).""" + + directory, name = shape + stems = [f"{stem}.yaml" for stem in _NAME_SPELLINGS[name]] + if directory is None: + return stems + return [ + f"{part}/{stem}" for part in _DIRECTORY_SPELLINGS[directory] for stem in stems + ] + + +@st.composite +def checkout_spellings(draw: st.DrawFn) -> tuple[list[str], list[str]]: + """Paths a commit records, and the paths then written, in writing order. + + Each recorded path is written as recorded, respelled within its group, or + not at all; up to two further paths are written too. Two recorded paths + can be spellings of one name. + """ + + recorded: dict[str, tuple[int | None, int]] = {} + for shape in draw(st.lists(_PATH_SHAPES, min_size=1, max_size=4)): + recorded.setdefault(draw(st.sampled_from(_spellings_of(shape))), shape) + written = [] + for path, shape in recorded.items(): + action = draw(st.sampled_from(("as recorded", "respelled", "absent"))) + if action == "as recorded": + written.append(path) + elif action == "respelled": + written.append(draw(st.sampled_from(_spellings_of(shape)))) + for shape in draw(st.lists(_PATH_SHAPES, max_size=2)): + written.append(draw(st.sampled_from(_spellings_of(shape)))) + return list(recorded), draw(st.permutations(written)) + + +def _pairs_one_to_one( + root: Path, recorded: Collection[str], present: Mapping[str, Path] +) -> bool: + """Brute force: can every recorded path take its own entry, using them all? + + A path may take an entry of the same name, or of a canonically equivalent + name when the path opens that entry's file. + """ + + def may_pair(path: str, entry: str) -> bool: + if path == entry: + return True + if unicodedata.normalize("NFD", path) != unicodedata.normalize("NFD", entry): + return False + try: + return (root / path).samefile(present[entry]) + except OSError: + return False + + paths = sorted(recorded) + if len(paths) != len(present): + return False + return any( + all(may_pair(path, entry) for path, entry in zip(paths, order, strict=True)) + for order in permutations(present) + ) + + +class TestMatchCheckoutPaths: + @pytest.mark.parametrize("composed_file", ["absent", "another file"]) + def test_an_equivalent_name_pairs_only_with_the_file_its_path_opens( + self, tmp_path, composed_file + ) -> None: + # An entry under an equivalent spelling pairs only when the recorded + # path opens that entry's file. ``present`` is built by hand, so the + # guard is exercised on every file system: on one that keeps NFC and + # NFD names apart, a lone NFD file is exactly the "absent" case. + (tmp_path / "other.yaml").write_bytes(b"other\n") + if composed_file == "another file": + (tmp_path / "caf\u00e9.yaml").write_bytes(b"composed\n") + present = {"cafe\u0301.yaml": tmp_path / "other.yaml"} + assert axiom_adapter._match_checkout_paths( + tmp_path, ["caf\u00e9.yaml"], present + ) == ({}, ["caf\u00e9.yaml"], ["cafe\u0301.yaml"]) + + def test_an_ambiguous_spelling_pairs_nothing(self, tmp_path) -> None: + # Two recorded spellings (NFC and the Angstrom sign) are equivalent to + # one NFD entry. Where the file system opens that file under both, either + # could take it; neither does, and the refusal names all three. + (tmp_path / "A\u030a.yaml").write_bytes(b"one file\n") + present = axiom_adapter._checkout_files(tmp_path) + assert axiom_adapter._match_checkout_paths( + tmp_path, ["\u212b.yaml", "\u00c5.yaml"], present + ) == ({}, sorted(["\u212b.yaml", "\u00c5.yaml"]), ["A\u030a.yaml"]) + + def test_distinct_nfc_and_nfd_files_are_not_merged(self, tmp_path) -> None: + composed, decomposed = "caf\u00e9.yaml", "cafe\u0301.yaml" + (tmp_path / composed).write_bytes(b"composed\n") + (tmp_path / decomposed).write_bytes(b"decomposed\n") + present = axiom_adapter._checkout_files(tmp_path) + if len(present) == 1: + pytest.skip( + "this file system opens one file under both spellings, so it " + "cannot hold two files whose names differ only in normalization" + ) + match = axiom_adapter._match_checkout_paths + files = {composed: tmp_path / composed, decomposed: tmp_path / decomposed} + assert match(tmp_path, [composed], present) == ( + {composed: files[composed]}, + [], + [decomposed], + ) + assert match(tmp_path, [decomposed], present) == ( + {decomposed: files[decomposed]}, + [], + [composed], + ) + assert match(tmp_path, [composed, decomposed], present) == (files, [], []) + + @settings(max_examples=150, deadline=None) + @given(spellings=checkout_spellings()) + def test_property_paths_pair_one_to_one_with_the_files_they_open( + self, tmp_path_factory, spellings + ) -> None: + """Invariant: the pairing is one to one and complete iff a pairing exists. + + The files are real, on the test's own file system: where it opens one + file under equivalent or case-differing names, writes collide as they + would in a checkout. Every pair is a path and an entry of the same + name, or of an equivalent name the path opens; no entry pairs twice; + the pairs and leftovers partition both sides; and nothing is left + over exactly when a brute-force search finds a complete pairing. + """ + + recorded, written = spellings + root = tmp_path_factory.mktemp("spellings") + for index, path in enumerate(written): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(b"%d\n" % index) + present = axiom_adapter._checkout_files(root) + paired, lacking, extra = axiom_adapter._match_checkout_paths( + root, recorded, present + ) + entry_of = {file: entry for entry, file in present.items()} + entries = [entry_of[file] for file in paired.values()] + assert len(set(entries)) == len(entries) + assert sorted([*paired, *lacking]) == sorted(recorded) + assert sorted([*entries, *extra]) == sorted(present) + for path, file in paired.items(): + entry = entry_of[file] + assert entry == path or ( + unicodedata.normalize("NFD", entry) + == unicodedata.normalize("NFD", path) + and (root / path).samefile(file) + ) + complete = not lacking and not extra + assert complete == _pairs_one_to_one(root, recorded, present) + event("complete" if complete else "refused") + respelled = sum(entry_of[file] != path for path, file in paired.items()) + event(f"pairs through an equivalent spelling: {respelled}") + + @pytest.fixture(scope="module") def blob_repositories(tmp_path_factory) -> dict[str, Path]: repositories = {} @@ -2014,13 +2303,21 @@ def _commit_submodule(root: Path, *, ignore: str | None = None) -> tuple[str, Pa def _old_mtime_checkout(root: Path) -> tuple[Path, str]: - """A committed checkout whose files all carry an old mtime.""" + """A committed checkout whose files all carry an old mtime. + + It holds ``_DECOMPOSED``, so on macOS, where git records that name in NFC + while the file system lists it in NFD, every check pairs the two. + """ - _write_tree(root, {**_BASE_TREE, ".gitignore": b"*.local.yaml\n"}) + _write_tree( + root, + {**_BASE_TREE, ".gitignore": b"*.local.yaml\n", _DECOMPOSED: b"rules: aux\n"}, + ) for path in root.rglob("*"): if path.is_file(): os.utime(path, (_OLD_MTIME, _OLD_MTIME)) _git(root, "init", "-q") + _git(root, "config", "core.precomposeunicode", "true") _git(root, "add", "-A") _git(root, "commit", "-q", "-m", "fixture") return root, _git(root, "rev-parse", "HEAD") @@ -2086,26 +2383,30 @@ def _commit_symlink_as_a_plain_file(root: Path) -> str: return _git(root, "rev-parse", "HEAD") -def _hide_a_same_size_edit_behind_a_clean_filter(root: Path) -> None: - """Edit the module, same size, behind a clean filter that undoes the edit.""" +def _hide_a_same_size_edit_behind_a_clean_filter( + root: Path, path: str = _MODULE +) -> None: + """Edit ``path``, same size, behind a clean filter that undoes the edit.""" (root / ".git" / "info").mkdir(exist_ok=True) (root / ".git" / "info" / "attributes").write_text( "*.yaml filter=restore\n", encoding="utf-8" ) _git(root, "config", "filter.restore.clean", "sed s/HIDDEN/rules:/") - module = root / _MODULE + module = root / path content = module.read_bytes() edited = content.replace(b"rules:", b"HIDDEN") assert len(edited) == len(content) and edited != content module.write_bytes(edited) -def _hide_an_executable_mode_change(root: Path, mode: int = 0o755) -> None: - """Change the module's executable bit while git ignores worktree modes.""" +def _hide_an_executable_mode_change( + root: Path, mode: int = 0o755, path: str = _MODULE +) -> None: + """Change ``path``'s executable bit while git ignores worktree modes.""" _git(root, "config", "core.filemode", "false") - (root / _MODULE).chmod(mode) + (root / path).chmod(mode) def _apply_checkout_change(root: Path, change: str, head: str, tmp_path_factory) -> str: